From 2443f2188e46c5393c7faa1c35fd3d217610bb17 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 5 Jan 2025 07:39:57 +0800 Subject: [PATCH 001/110] Add new configure.ac option --enable-tail-call-interp --- configure | 29 +++++++++++++++++++++++++++++ configure.ac | 16 ++++++++++++++++ pyconfig.h.in | 3 +++ 3 files changed, 48 insertions(+) diff --git a/configure b/configure index aa88c74c61156c..daeb6fc8db772f 100755 --- a/configure +++ b/configure @@ -1083,6 +1083,7 @@ enable_shared with_static_libpython enable_profiling enable_gil +enable_tail_call_interp with_pydebug with_trace_refs enable_pystats @@ -1814,6 +1815,8 @@ Optional Features: no) --disable-gil enable experimental support for running without the GIL (default is no) + --enable-tail-call-interp + enable tail calling interpreter --enable-pystats enable internal statistics gathering (default is no) --enable-experimental-jit[=no|yes|yes-off|interpreter] build the experimental just-in-time compiler @@ -8231,6 +8234,32 @@ printf "%s\n" "#define Py_GIL_DISABLED 1" >>confdefs.h ABI_THREAD="t" fi +# Check for --enable-tail-call-interp +# --enable-tail-call-interp +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-tail-call-interp" >&5 +printf %s "checking for --enable-tail-call-interp... " >&6; } +# Check whether --enable-tail-call-interp was given. +if test ${enable_tail_call_interp+y} +then : + enableval=$enable_tail_call_interp; +if test "$enableval" != no +then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; }; + +printf "%s\n" "#define Py_TAIL_CALL_INTERP 1" >>confdefs.h + +else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; }; +fi +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } ;; +esac +fi + + # Check for --with-pydebug { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-pydebug" >&5 printf %s "checking for --with-pydebug... " >&6; } diff --git a/configure.ac b/configure.ac index 9e131ed1a2dc98..6a38b673f8a3f2 100644 --- a/configure.ac +++ b/configure.ac @@ -1717,6 +1717,22 @@ then ABI_THREAD="t" fi +# Check for --enable-tail-call-interp +# --enable-tail-call-interp +AC_MSG_CHECKING([for --enable-tail-call-interp]) +AC_ARG_ENABLE([tail-call-interp], + [AS_HELP_STRING([--enable-tail-call-interp], [enable tail calling interpreter]) ], +[ +if test "$enableval" != no +then + AC_MSG_RESULT([yes]); + AC_DEFINE([Py_TAIL_CALL_INTERP], [1], + [Define if you want to enable tail calling interpreter]) +else + AC_MSG_RESULT([no]); +fi], +[AC_MSG_RESULT([no])]) + # Check for --with-pydebug AC_MSG_CHECKING([for --with-pydebug]) AC_ARG_WITH([pydebug], diff --git a/pyconfig.h.in b/pyconfig.h.in index d862966b7de4c8..1ee381d98d4fb8 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1718,6 +1718,9 @@ /* The version of SunOS/Solaris as reported by `uname -r' without the dot. */ #undef Py_SUNOS_VERSION +/* Define if you want to enable tail calling interpreter */ +#undef Py_TAIL_CALL_INTERP + /* Define if you want to enable tracing references for debugging purpose */ #undef Py_TRACE_REFS From fb72fcdac4e48e41999b8e631e738ae4b6c9b4e1 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 5 Jan 2025 10:05:04 +0800 Subject: [PATCH 002/110] sorta working --- Python/bytecodes.c | 10 +- Python/ceval.c | 82 +- Python/ceval_macros.h | 81 +- Python/generated_cases_tail_call.c.h | 33746 ++++++++++++++++ Tools/cases_generator/tier1_generator.py | 65 +- .../tier1_tail_call_generator.py | 235 + 6 files changed, 34157 insertions(+), 62 deletions(-) create mode 100644 Python/generated_cases_tail_call.c.h create mode 100644 Tools/cases_generator/tier1_tail_call_generator.py diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 602cf7f47b812b..facf28ad0a831d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1018,7 +1018,7 @@ dummy_func( } tier1 inst(INTERPRETER_EXIT, (retval --)) { - assert(frame == &entry_frame); + assert(frame == entry_frame); assert(_PyFrame_IsIncomplete(frame)); /* Restore previous frame and return. */ tstate->current_frame = frame->previous; @@ -1034,7 +1034,7 @@ dummy_func( // is pushed to a different frame, the callers' frame. inst(RETURN_VALUE, (retval -- res)) { #if TIER_ONE - assert(frame != &entry_frame); + assert(frame != entry_frame); #endif _PyStackRef temp = retval; DEAD(retval); @@ -1133,7 +1133,7 @@ dummy_func( PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); PyObject *retval_o; - assert(frame != &entry_frame); + assert(frame != entry_frame); if ((tstate->interp->eval_frame == NULL) && (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) @@ -1207,7 +1207,7 @@ dummy_func( // The compiler treats any exception raised here as a failed close() // or throw() call. #if TIER_ONE - assert(frame != &entry_frame); + assert(frame != entry_frame); #endif frame->instr_ptr++; PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); @@ -1303,7 +1303,7 @@ dummy_func( tier1 inst(CLEANUP_THROW, (sub_iter_st, last_sent_val_st, exc_value_st -- none, value)) { PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - assert(throwflag); + // assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); diff --git a/Python/ceval.c b/Python/ceval.c index e92a11b16cec81..1f7515e22a52dc 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -781,13 +781,33 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) /* This setting is reversed below following _PyEval_EvalFrameDefault */ #endif +#ifdef Py_TAIL_CALL_INTERP +#include "generated_cases_tail_call.c.h" +#endif + +#ifdef LLTRACE +PyObject * +_TAIL_CALL_shim(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +{ + return (INSTRUCTION_TABLE[next_instr->op.code])(frame, stack_pointer, tstate, next_instr, next_instr->op.arg, entry_frame, lltrace); +} +#else +PyObject * +_TAIL_CALL_shim(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +{ + return (INSTRUCTION_TABLE[next_instr->op.code])(frame, stack_pointer, tstate, next_instr, next_instr->op.arg, entry_frame); +} +#endif + PyObject* _Py_HOT_FUNCTION _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { _Py_EnsureTstateNotNULL(tstate); CALL_STAT_INC(pyeval_calls); -#if USE_COMPUTED_GOTOS +#if USE_COMPUTED_GOTOS && !defined(Py_TAIL_CALL_INTERP) /* Import the static jump table */ #include "opcode_targets.h" #endif @@ -801,27 +821,28 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int int lltrace = 0; #endif - _PyInterpreterFrame entry_frame; + _PyInterpreterFrame e; + _PyInterpreterFrame *entry_frame = &e; #if defined(Py_DEBUG) && !defined(Py_STACKREF_DEBUG) /* Set these to invalid but identifiable values for debugging. */ - entry_frame.f_funcobj = (_PyStackRef){.bits = 0xaaa0}; - entry_frame.f_locals = (PyObject*)0xaaa1; - entry_frame.frame_obj = (PyFrameObject*)0xaaa2; - entry_frame.f_globals = (PyObject*)0xaaa3; - entry_frame.f_builtins = (PyObject*)0xaaa4; + e.f_funcobj = (_PyStackRef){.bits = 0xaaa0}; + e.f_locals = (PyObject*)0xaaa1; + e.frame_obj = (PyFrameObject*)0xaaa2; + e.f_globals = (PyObject*)0xaaa3; + e.f_builtins = (PyObject*)0xaaa4; #endif - entry_frame.f_executable = PyStackRef_None; - entry_frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; - entry_frame.stackpointer = entry_frame.localsplus; - entry_frame.owner = FRAME_OWNED_BY_CSTACK; - entry_frame.visited = 0; - entry_frame.return_offset = 0; + e.f_executable = PyStackRef_None; + e.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; + e.stackpointer = e.localsplus; + e.owner = FRAME_OWNED_BY_CSTACK; + e.visited = 0; + e.return_offset = 0; /* Push frame */ - entry_frame.previous = tstate->current_frame; - frame->previous = &entry_frame; + e.previous = tstate->current_frame; + frame->previous = entry_frame; tstate->current_frame = frame; tstate->c_recursion_remaining -= (PY_EVAL_C_STACK_UNITS - 1); @@ -878,7 +899,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int stack_pointer = _PyFrame_GetStackPointer(frame); #ifdef LLTRACE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + lltrace = maybe_lltrace_resume_frame(frame, entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } @@ -891,18 +912,26 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int assert(!_PyErr_Occurred(tstate)); #endif +#ifdef Py_TAIL_CALL_INTERP +#ifdef LLTRACE + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, entry_frame, lltrace); +#else + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, entry_frame); +#endif +#else DISPATCH(); +#endif { /* Start instructions */ -#if !USE_COMPUTED_GOTOS +#if !USE_COMPUTED_GOTOS && !defined(Py_TAIL_CALL_INTERP) dispatch_opcode: switch (opcode) #endif { - +#ifndef Py_TAIL_CALL_INTERP #include "generated_cases.c.h" - +#endif #if USE_COMPUTED_GOTOS _unknown_opcode: @@ -945,7 +974,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif /* Log traceback info. */ - assert(frame != &entry_frame); + assert(frame != entry_frame); if (!_PyFrame_IsIncomplete(frame)) { PyFrameObject *f = _PyFrame_GetFrameObject(frame); if (f != NULL) { @@ -1004,20 +1033,29 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int lltrace_resume_frame(frame); } #endif + +#ifdef Py_TAIL_CALL_INTERP +#ifdef LLTRACE + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, entry_frame, lltrace); +#else + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, entry_frame); +#endif +#else DISPATCH(); +#endif } } exit_unwind: assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); + assert(frame != entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; - if (frame == &entry_frame) { + if (frame == entry_frame) { /* Restore previous frame and exit */ tstate->current_frame = frame->previous; tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index f15633fa467376..bd57ffea4133af 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -70,7 +70,26 @@ #define INSTRUCTION_STATS(op) ((void)0) #endif -#if USE_COMPUTED_GOTOS +#ifdef Py_TAIL_CALL_INTERP +#ifdef LLTRACE +__attribute__((preserve_none)) +typedef PyObject* (*py_tail_call_funcptr)(_PyInterpreterFrame *, _PyStackRef *, PyThreadState *tstate, _Py_CODEUNIT *, int, _PyInterpreterFrame *, int); +#else +__attribute__((preserve_none)) +typedef PyObject* (*py_tail_call_funcptr)(_PyInterpreterFrame *, _PyStackRef *, PyThreadState *tstate, _Py_CODEUNIT *, int, _PyInterpreterFrame *); +#endif +#ifdef LLTRACE +#define DISPATCH_GOTO() do { \ + __attribute__((musttail)) \ + return (INSTRUCTION_TABLE[opcode])(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); \ +} while (0) +#else +#define DISPATCH_GOTO() do { \ + __attribute__((musttail)) \ + return (INSTRUCTION_TABLE[opcode])(frame, stack_pointer, tstate, next_instr, entry_frame, oparg); \ +} while (0) +#endif +#elif USE_COMPUTED_GOTOS # define TARGET(op) TARGET_##op: # define DISPATCH_GOTO() goto *opcode_targets[opcode] #else @@ -89,7 +108,7 @@ #if LLTRACE #define LLTRACE_RESUME_FRAME() \ do { \ - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); \ + lltrace = maybe_lltrace_resume_frame(frame, entry_frame, GLOBALS()); \ if (lltrace < 0) { \ goto exit_unwind; \ } \ @@ -121,6 +140,46 @@ do { \ DISPATCH_GOTO(); \ } +#ifdef Py_TAIL_CALL_INTERP +#ifdef LLTRACE +#define DISPATCH_INLINED(NEW_FRAME) \ + do { \ + assert(tstate->interp->eval_frame == NULL); \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + assert((NEW_FRAME)->previous == frame); \ + frame = tstate->current_frame = (NEW_FRAME); \ + CALL_STAT_INC(inlined_py_calls); \ + if (_Py_EnterRecursivePy(tstate)) {\ + goto exit_unwind;\ + } \ + next_instr = frame->instr_ptr; \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + lltrace = maybe_lltrace_resume_frame(frame, entry_frame, GLOBALS()); \ + if (lltrace < 0) { \ + goto exit_unwind; \ + } \ + NEXTOPARG(); \ + __attribute__((musttail)) \ + return (INSTRUCTION_TABLE[opcode])(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); \ + } while (0) +#else +do { \ + assert(tstate->interp->eval_frame == NULL); \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + assert((NEW_FRAME)->previous == frame); \ + frame = tstate->current_frame = (NEW_FRAME); \ + CALL_STAT_INC(inlined_py_calls); \ + if (_Py_EnterRecursivePy(tstate)) {\ + goto exit_unwind;\ + } \ + next_instr = frame->instr_ptr; \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + NEXTOPARG(); \ + __attribute__((musttail)) \ + return (INSTRUCTION_TABLE[opcode])(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); \ + } while (0) +#endif +#else #define DISPATCH_INLINED(NEW_FRAME) \ do { \ assert(tstate->interp->eval_frame == NULL); \ @@ -130,6 +189,7 @@ do { \ CALL_STAT_INC(inlined_py_calls); \ goto start_frame; \ } while (0) +#endif // Use this instead of 'goto error' so Tier 2 can go to a different label #define GOTO_ERROR(LABEL) goto LABEL @@ -238,7 +298,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { #endif #define WITHIN_STACK_BOUNDS() \ - (frame == &entry_frame || (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE())) + (frame == entry_frame || (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE())) /* Data access macros */ #define FRAME_CO_CONSTS (_PyFrame_GetCode(frame)->co_consts) @@ -258,8 +318,21 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define SETLOCAL(i, value) do { _PyStackRef tmp = GETLOCAL(i); \ GETLOCAL(i) = value; \ PyStackRef_XCLOSE(tmp); } while (0) - +#ifdef Py_TAIL_CALL_INTERP +#ifdef LLTRACE +#define GO_TO_INSTRUCTION(op) do { \ + __attribute__((musttail)) \ + return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - _PyOpcode_Caches[_PyOpcode_Deopt[op]], oparg, entry_frame, lltrace); \ +} while (0) +#else +#define GO_TO_INSTRUCTION(op) do { \ + __attribute__((musttail)) \ + return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - _PyOpcode_Caches[_PyOpcode_Deopt[op]], oparg, entry_frame); \ +} while (0) +#endif +#else #define GO_TO_INSTRUCTION(op) goto PREDICT_ID(op) +#endif #ifdef Py_STATS #define UPDATE_MISS_STATS(INSTNAME) \ diff --git a/Python/generated_cases_tail_call.c.h b/Python/generated_cases_tail_call.c.h new file mode 100644 index 00000000000000..59d4bafd20a556 --- /dev/null +++ b/Python/generated_cases_tail_call.c.h @@ -0,0 +1,33746 @@ +// This file is generated by Tools/cases_generator/tier1_tail_call_generator.py +// from: +// Python/bytecodes.c +// Do not edit! + +#ifndef Py_TAIL_CALL_INTERP + #error "This file is for tail-calling interpreter only." +#endif +#define TIER_ONE 1 +static py_tail_call_funcptr INSTRUCTION_TABLE[256]; + + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP); + PREDICTED(BINARY_OP); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef lhs; + _PyStackRef rhs; + _PyStackRef res; + // _SPECIALIZE_BINARY_OP + { + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(BINARY_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + assert(NB_ADD <= oparg); + assert(oparg <= NB_INPLACE_XOR); + } + // _BINARY_OP + { + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); + assert(_PyEval_BinaryOps[oparg]); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(lhs); + PyStackRef_CLOSE(rhs); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP_ADD_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP_ADD_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_FLOAT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_ADD_FLOAT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP_ADD_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP_ADD_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_INT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_ADD_INT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_UNICODE + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_ADD_UNICODE + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + // _GUARD_BOTH_UNICODE + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_INPLACE_ADD_UNICODE + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + int next_oparg; + #if TIER_ONE + assert(next_instr->op.code == STORE_FAST); + next_oparg = next_instr->op.arg; + #else + next_oparg = CURRENT_OPERAND0(); + #endif + _PyStackRef *target_local = &GETLOCAL(next_oparg); + DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP); + STAT_INC(BINARY_OP, hit); + /* Handle `left = left + right` or `left += right` for str. + * + * When possible, extend `left` in place rather than + * allocating a new PyUnicodeObject. This attempts to avoid + * quadratic behavior when one neglects to use str.join(). + * + * If `left` has only two references remaining (one from + * the stack, one in the locals), DECREFing `left` leaves + * only the locals reference, so PyUnicode_Append knows + * that the string is safe to mutate. + */ + assert(Py_REFCNT(left_o) >= 2); + PyStackRef_CLOSE(left); + PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); + PyUnicode_Append(&temp, right_o); + *target_local = PyStackRef_FromPyObjectSteal(temp); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (PyStackRef_IsNull(*target_local)) goto pop_2_error; + #if TIER_ONE + // The STORE_FAST is already done. This is done here in tier one, + // and during trace projection in tier two: + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); + #endif + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_FLOAT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_MULTIPLY_FLOAT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP_MULTIPLY_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP_MULTIPLY_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_INT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_MULTIPLY_INT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_FLOAT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_SUBTRACT_FLOAT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP_SUBTRACT_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP_SUBTRACT_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_INT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_SUBTRACT_INT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BINARY_SLICE); + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + _PyStackRef res; + // _SPECIALIZE_BINARY_SLICE + { + // Placeholder until we implement BINARY_SLICE specialization + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(BINARY_SLICE); + #endif /* ENABLE_SPECIALIZATION */ + } + // _BINARY_SLICE + { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyObject *res_o; + // Can't use ERROR_IF() here, because we haven't + // DECREF'ed container yet, and we still own slice. + if (slice == NULL) { + res_o = NULL; + } + else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(slice); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } + PyStackRef_CLOSE(container); + if (res_o == NULL) goto pop_3_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR); + PREDICTED(BINARY_SUBSCR); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef container; + _PyStackRef sub; + _PyStackRef res; + // _SPECIALIZE_BINARY_SUBSCR + { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + assert(frame->stackpointer == NULL); + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_BinarySubscr(container, sub, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(BINARY_SUBSCR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _BINARY_SUBSCR + { + PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); + PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_GetItem(container_o, sub_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef dict_st; + _PyStackRef sub_st; + _PyStackRef res; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + dict_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int rc = PyDict_GetItemRef(dict, sub, &res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (rc == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetKeyError(sub); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + PyStackRef_CLOSE(dict_st); + PyStackRef_CLOSE(sub_st); + if (rc <= 0) goto pop_2_error; + // not found or error + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_SUBSCR_GETITEM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_SUBSCR_GETITEM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef container; + _PyStackRef getitem; + _PyStackRef sub; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + } + // _BINARY_SUBSCR_CHECK_FUNC + { + container = stack_pointer[-2]; + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); + DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR); + assert(PyFunction_Check(getitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); + DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR); + PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); + assert(code->co_argcount == 2); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + getitem = PyStackRef_FromPyObjectNew(getitem_o); + STAT_INC(BINARY_SUBSCR, hit); + } + // _BINARY_SUBSCR_INIT_CALL + { + sub = stack_pointer[-1]; + new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + new_frame->localsplus[0] = container; + new_frame->localsplus[1] = sub; + frame->return_offset = 2 ; + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef list_st; + _PyStackRef sub_st; + _PyStackRef res; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyList_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + stack_pointer = _PyFrame_GetStackPointer(frame); + DEOPT_IF(res_o == NULL, BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + #else + DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + Py_INCREF(res_o); + #endif + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(list_st); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_SUBSCR_STR_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_SUBSCR_STR_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef str_st; + _PyStackRef sub_st; + _PyStackRef res; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + str_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + // Specialize for reading an ASCII character from any string: + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(str_st); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef tuple_st; + _PyStackRef sub_st; + _PyStackRef res; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + tuple_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyTuple_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + Py_INCREF(res_o); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(tuple_st); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BUILD_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BUILD_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_LIST); + _PyStackRef *values; + _PyStackRef list; + values = &stack_pointer[-oparg]; + PyObject *list_o = _PyList_FromStackRefSteal(values, oparg); + if (list_o == NULL) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + list = PyStackRef_FromPyObjectSteal(list_o); + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BUILD_MAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BUILD_MAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_MAP); + _PyStackRef *values; + _PyStackRef map; + values = &stack_pointer[-oparg*2]; + STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); + if (CONVERSION_FAILED(values_o)) { + for (int _i = oparg*2; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + { + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *map_o = _PyDict_FromItems( + values_o, 2, + values_o+1, 2, + oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + for (int _i = oparg*2; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + if (map_o == NULL) { + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + map = PyStackRef_FromPyObjectSteal(map_o); + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BUILD_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BUILD_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SET); + _PyStackRef *values; + _PyStackRef set; + values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *set_o = PySet_New(NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (set_o == NULL) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + int err = 0; + for (int i = 0; i < oparg; i++) { + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PySet_Add(set_o, PyStackRef_AsPyObjectBorrow(values[i])); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + PyStackRef_CLOSE(values[i]); + } + if (err != 0) { + Py_DECREF(set_o); + { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + set = PyStackRef_FromPyObjectSteal(set_o); + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BUILD_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BUILD_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SLICE); + _PyStackRef start; + _PyStackRef stop; + _PyStackRef step = PyStackRef_NULL; + _PyStackRef slice; + if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } + stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; + start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; + PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); + PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); + PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); + PyStackRef_CLOSE(start); + PyStackRef_CLOSE(stop); + PyStackRef_XCLOSE(step); + if (slice_o == NULL) { + stack_pointer += -2 - ((oparg == 3) ? 1 : 0); + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + slice = PyStackRef_FromPyObjectSteal(slice_o); + stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; + stack_pointer += -1 - ((oparg == 3) ? 1 : 0); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BUILD_STRING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BUILD_STRING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_STRING); + _PyStackRef *pieces; + _PyStackRef str; + pieces = &stack_pointer[-oparg]; + STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); + if (CONVERSION_FAILED(pieces_o)) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(pieces[_i]); + } + { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(pieces[_i]); + } + if (str_o == NULL) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + str = PyStackRef_FromPyObjectSteal(str_o); + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BUILD_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BUILD_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TUPLE); + _PyStackRef *values; + _PyStackRef tup; + values = &stack_pointer[-oparg]; + PyObject *tup_o = _PyTuple_FromStackRefSteal(values, oparg); + if (tup_o == NULL) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + tup = PyStackRef_FromPyObjectSteal(tup_o); + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CACHE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CACHE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CACHE); + assert(0 && "Executing a cache."); + Py_FatalError("Executing a cache."); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL); + PREDICTED(CALL); + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef res; + // _SPECIALIZE_CALL + { + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CALL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD + { + args = &stack_pointer[-oparg]; + func = &stack_pointer[-2 - oparg]; + maybe_self = &stack_pointer[-1 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); + } + } + // _DO_CALL + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; + } + frame->return_offset = 4 ; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(res_o); + } + } + } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef *args; + _PyStackRef *init; + _PyStackRef *self; + _PyInterpreterFrame *init_frame; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_AND_ALLOCATE_OBJECT + { + args = &stack_pointer[-oparg]; + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + init = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL); + PyTypeObject *tp = (PyTypeObject *)callable_o; + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL); + assert(tp->tp_new == PyBaseObject_Type.tp_new); + assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_alloc == PyType_GenericAlloc); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; + PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); + PyCodeObject *code = (PyCodeObject *)init_func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *self_o = PyType_GenericAlloc(tp, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (self_o == NULL) { + goto error; + } + self[0] = PyStackRef_FromPyObjectSteal(self_o); + _PyStackRef temp = callable[0]; + init[0] = PyStackRef_FromPyObjectNew(init_func); + PyStackRef_CLOSE(temp); + } + // _CREATE_INIT_FRAME + { + args = &stack_pointer[-oparg]; + self = &stack_pointer[-1 - oparg]; + init = &stack_pointer[-2 - oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); + stack_pointer = _PyFrame_GetStackPointer(frame); + /* Push self onto stack of shim */ + shim->localsplus[0] = PyStackRef_DUP(self[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, init[0], NULL, args-1, oparg+1, NULL, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + _PyEval_FrameClearAndPop(tstate, shim); + goto error; + } + init_frame = temp; + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; + /* Account for pushing the extra frame. + * We don't check recursion depth here, + * as it will be checked after start_frame */ + tstate->py_recursion_remaining--; + } + // _PUSH_FRAME + { + new_frame = init_frame; + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef *func; + _PyStackRef *self; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + { + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL); + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + func = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + STAT_INC(CALL, hit); + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + PyStackRef_CLOSE(temp); + } + // flush + // _CHECK_FUNCTION_VERSION + { + callable = &stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL); + } + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null[0]); + STAT_INC(CALL, hit); + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null[0]; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef *method; + _PyStackRef *self; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_METHOD_VERSION + { + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL); + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + DEOPT_IF(!PyFunction_Check(func), CALL); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + } + // _EXPAND_METHOD + { + method = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + assert(PyStackRef_IsNull(null[0])); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable[0]; + method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(method[0])); + PyStackRef_CLOSE(temp); + } + // flush + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + // The frame has stolen all the arguments from the stack. + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + goto error; + } + new_frame = temp; + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_BUILTIN_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_BUILTIN_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_CLASS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_CLASS + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + DEOPT_IF(!PyType_Check(callable_o), CALL); + PyTypeObject *tp = (PyTypeObject *)callable_o; + DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + STAT_INC(CALL, hit); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_BUILTIN_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_BUILTIN_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* Builtin METH_FASTCALL functions, without keywords */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + /* res = func(self, args, nargs) */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable_o), + args_o, + total_args); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); + STAT_INC(CALL, hit); + /* res = func(self, args, nargs, kwnames) */ + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_BUILTIN_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_BUILTIN_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_O + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* Builtin METH_O functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + _PyStackRef arg = args[0]; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(arg); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_FUNCTION_EX); + PREDICTED(CALL_FUNCTION_EX); + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef kwargs_in = PyStackRef_NULL; + _PyStackRef tuple; + _PyStackRef kwargs_out = PyStackRef_NULL; + _PyStackRef func_st; + _PyStackRef callargs_st; + _PyStackRef kwargs_st = PyStackRef_NULL; + _PyStackRef result; + // _MAKE_CALLARGS_A_TUPLE + { + if (oparg & 1) { kwargs_in = stack_pointer[-(oparg & 1)]; } + callargs = stack_pointer[-1 - (oparg & 1)]; + func = stack_pointer[-3 - (oparg & 1)]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (PyTuple_CheckExact(callargs_o)) { + tuple = callargs; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { + goto error; + } + PyStackRef_CLOSE(callargs); + tuple = PyStackRef_FromPyObjectSteal(tuple_o); + } + kwargs_out = kwargs_in; + } + // _DO_CALL_FUNCTION_EX + { + kwargs_st = kwargs_out; + callargs_st = tuple; + func_st = func; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-1 - (oparg & 1)] = callargs_st; + if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(result_o); + } + } + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer += -2 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Need to sync the stack since we exit with DISPATCH_INLINED. + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + goto error; + } + assert( 1 == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + stack_pointer[-1 - (oparg & 1)] = callargs_st; + if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(kwargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(callargs_st); + PyStackRef_CLOSE(func_st); + if (result_o == NULL) { + stack_pointer += -3 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + result = PyStackRef_FromPyObjectSteal(result_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3 - (oparg & 1)] = result; + stack_pointer += -2 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 2 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-3 - (oparg & 1)] = result; + stack_pointer += -2 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_INTRINSIC_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_INTRINSIC_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_1); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(oparg <= MAX_INTRINSIC_1); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) goto pop_1_error; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-1] = res; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_INTRINSIC_2(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_INTRINSIC_2(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_2); + _PyStackRef value2_st; + _PyStackRef value1_st; + _PyStackRef res; + value1_st = stack_pointer[-1]; + value2_st = stack_pointer[-2]; + assert(oparg <= MAX_INTRINSIC_2); + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value2_st); + PyStackRef_CLOSE(value1_st); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_ISINSTANCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_ISINSTANCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ISINSTANCE); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* isinstance(o, o2) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + DEOPT_IF(total_args != 2, CALL); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL); + STAT_INC(CALL, hit); + _PyStackRef cls_stackref = args[1]; + _PyStackRef inst_stackref = args[0]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (retval < 0) { + goto error; + } + res = retval ? PyStackRef_True : PyStackRef_False; + assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(inst_stackref); + PyStackRef_CLOSE(cls_stackref); + PyStackRef_CLOSE(callable[0]); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW); + PREDICTED(CALL_KW); + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef kwnames_in; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef kwnames_out; + _PyStackRef res; + // _SPECIALIZE_CALL_KW + { + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CALL_KW); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + kwnames_in = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + func = &stack_pointer[-3 - oparg]; + maybe_self = &stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); + } + kwnames_out = kwnames_in; + } + // _DO_CALL_KW + { + kwnames = kwnames_out; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-1] = kwnames; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(kwnames); + // Sync stack explicitly since we leave using DISPATCH_INLINED(). + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; + } + assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4 ; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + stack_pointer[-1] = kwnames; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(res_o); + } + } + } + PyStackRef_CLOSE(kwnames); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + if (res_o == NULL) { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_KW_BOUND_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_KW_BOUND_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef kwnames; + _PyStackRef *method; + _PyStackRef *self; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + } + // _CHECK_METHOD_VERSION_KW + { + null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW); + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + DEOPT_IF(!PyFunction_Check(func), CALL_KW); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW); + } + // _EXPAND_METHOD_KW + { + method = &stack_pointer[-3 - oparg]; + self = &stack_pointer[-2 - oparg]; + _PyStackRef callable_s = callable[0]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable_s); + assert(PyStackRef_IsNull(null[0])); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(method[0])); + PyStackRef_CLOSE(callable_s); + } + // flush + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(kwnames); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + goto error; + } + new_frame = temp; + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_KW_NON_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_KW_NON_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_NON_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef kwnames; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE_KW + { + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(PyFunction_Check(callable_o), CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW); + } + // _CALL_KW_NON_PY + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(kwnames); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 2 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_KW_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_KW_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef kwnames; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + } + // _CHECK_FUNCTION_VERSION_KW + { + callable = &stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL_KW); + } + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(kwnames); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + goto error; + } + new_frame = temp; + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LEN); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* len(o) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable_o != interp->callable_cache.len, CALL); + STAT_INC(CALL, hit); + _PyStackRef arg_stackref = args[0]; + PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (len_i < 0) { + goto error; + } + PyObject *res_o = PyLong_FromSsize_t(len_i); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + GOTO_ERROR(error); + } + PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(arg_stackref); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LIST_APPEND); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self; + _PyStackRef arg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + arg = stack_pointer[-1]; + self = stack_pointer[-2]; + callable = stack_pointer[-3]; + assert(oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL); + assert(self_o != NULL); + DEOPT_IF(!PyList_Check(self_o), CALL); + DEOPT_IF(!LOCK_OBJECT(self_o), CALL); + STAT_INC(CALL, hit); + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); + PyStackRef_CLOSE(self); + PyStackRef_CLOSE(callable); + if (err) goto pop_3_error; + #if TIER_ONE + // Skip the following POP_TOP. This is done here in tier one, and + // during trace projection in tier two: + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); + #endif + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + /* Builtin METH_FASTCALL methods, without keywords */ + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + int nargs = total_args - 1; + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFast cfunc = + (PyCFunctionFast)(void(*)(void))meth->ml_meth; + PyObject *res_o = cfunc(self, (args_o + 1), nargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Clear the stack of the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + PyTypeObject *d_type = method->d_common.d_type; + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + STAT_INC(CALL, hit); + int nargs = total_args - 1; + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_NOARGS + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + assert(oparg == 0 || oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_O + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_O, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + _PyStackRef arg_stackref = args[1]; + _PyStackRef self_stackref = args[0]; + DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, + PyStackRef_AsPyObjectBorrow(self_stackref), + PyStackRef_AsPyObjectBorrow(arg_stackref)); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(arg_stackref); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_NON_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_NON_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_NON_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE + { + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(PyFunction_Check(callable_o), CALL); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL); + } + // _CALL_NON_PY_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_PY_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_PY_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_FUNCTION_VERSION + { + callable = &stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL); + } + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null[0]); + STAT_INC(CALL, hit); + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null[0]; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_FUNCTION_VERSION + { + callable = &stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL); + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + // The frame has stolen all the arguments from the stack. + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + goto error; + } + new_frame = temp; + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_STR_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_STR_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_STR_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_STR_1 + { + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Str(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(arg); + if (res_o == NULL) goto pop_3_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_TUPLE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_TUPLE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TUPLE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_TUPLE_1 + { + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PySequence_Tuple(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(arg); + if (res_o == NULL) goto pop_3_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_TYPE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_TYPE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TYPE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL); + STAT_INC(CALL, hit); + res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); + PyStackRef_CLOSE(arg); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CHECK_EG_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CHECK_EG_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EG_MATCH); + _PyStackRef exc_value_st; + _PyStackRef match_type_st; + _PyStackRef rest; + _PyStackRef match; + match_type_st = stack_pointer[-1]; + exc_value_st = stack_pointer[-2]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + PyStackRef_CLOSE(exc_value_st); + PyStackRef_CLOSE(match_type_st); + goto pop_2_error; + } + PyObject *match_o = NULL; + PyObject *rest_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, + &match_o, &rest_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(exc_value_st); + PyStackRef_CLOSE(match_type_st); + if (res < 0) goto pop_2_error; + assert((match_o == NULL) == (rest_o == NULL)); + if (match_o == NULL) goto pop_2_error; + if (!Py_IsNone(match_o)) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyErr_SetHandledException(match_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } + rest = PyStackRef_FromPyObjectSteal(rest_o); + match = PyStackRef_FromPyObjectSteal(match_o); + stack_pointer[-2] = rest; + stack_pointer[-1] = match; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CHECK_EXC_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CHECK_EXC_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EXC_MATCH); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyExceptionInstance_Check(left_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptTypeValid(tstate, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + PyStackRef_CLOSE(right); + goto pop_1_error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyErr_GivenExceptionMatches(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(right); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[-1] = b; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CLEANUP_THROW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CLEANUP_THROW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(CLEANUP_THROW); + _PyStackRef sub_iter_st; + _PyStackRef last_sent_val_st; + _PyStackRef exc_value_st; + _PyStackRef none; + _PyStackRef value; + exc_value_st = stack_pointer[-1]; + last_sent_val_st = stack_pointer[-2]; + sub_iter_st = stack_pointer[-3]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + // assert(throwflag); + assert(exc_value && PyExceptionInstance_Check(exc_value)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + none = PyStackRef_None; + value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); + PyStackRef_CLOSE(sub_iter_st); + PyStackRef_CLOSE(last_sent_val_st); + PyStackRef_CLOSE(exc_value_st); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto exception_unwind; + } + stack_pointer[-3] = none; + stack_pointer[-2] = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_COMPARE_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_COMPARE_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP); + PREDICTED(COMPARE_OP); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _SPECIALIZE_COMPARE_OP + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CompareOp(left, right, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(COMPARE_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + // _COMPARE_OP + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert((oparg >> 5) <= Py_GE); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res_o == NULL) goto pop_2_error; + if (oparg & 16) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res_bool = PyObject_IsTrue(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(res_o); + if (res_bool < 0) goto error; + res = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_COMPARE_OP_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_COMPARE_OP_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_FLOAT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP); + } + /* Skip 1 cache entry */ + // _COMPARE_OP_FLOAT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); + // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg + int sign_ish = COMPARISON_BIT(dleft, dright); + PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + // It's always a bool, so we don't care about oparg & 16. + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_COMPARE_OP_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_COMPARE_OP_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_INT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_INT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP); + } + /* Skip 1 cache entry */ + // _COMPARE_OP_INT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && + _PyLong_DigitCount((PyLongObject *)right_o) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); + // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg + int sign_ish = COMPARISON_BIT(ileft, iright); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + // It's always a bool, so we don't care about oparg & 16. + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_COMPARE_OP_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_COMPARE_OP_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_STR); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_UNICODE + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP); + } + /* Skip 1 cache entry */ + // _COMPARE_OP_STR + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left_o, right_o); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; + // It's always a bool, so we don't care about oparg & 16. + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CONTAINS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CONTAINS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP); + PREDICTED(CONTAINS_OP); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + // _SPECIALIZE_CONTAINS_OP + { + right = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ContainsOp(right, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CONTAINS_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CONTAINS_OP + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PySequence_Contains(right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) goto pop_2_error; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CONTAINS_OP_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CONTAINS_OP_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_DICT); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + /* Skip 1 cache entry */ + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP); + STAT_INC(CONTAINS_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyDict_Contains(right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) goto pop_2_error; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CONTAINS_OP_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CONTAINS_OP_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_SET); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + /* Skip 1 cache entry */ + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP); + STAT_INC(CONTAINS_OP, hit); + // Note: both set and frozenset use the same seq_contains method! + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PySet_Contains((PySetObject *)right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) goto pop_2_error; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CONVERT_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CONVERT_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONVERT_VALUE); + _PyStackRef value; + _PyStackRef result; + value = stack_pointer[-1]; + conversion_func conv_fn; + assert(oparg >= FVC_STR && oparg <= FVC_ASCII); + conv_fn = _PyEval_ConversionFuncs[oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (result_o == NULL) goto pop_1_error; + result = PyStackRef_FromPyObjectSteal(result_o); + stack_pointer[-1] = result; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_COPY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_COPY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY); + _PyStackRef bottom; + _PyStackRef top; + bottom = stack_pointer[-1 - (oparg-1)]; + assert(oparg > 0); + top = PyStackRef_DUP(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_COPY_FREE_VARS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_COPY_FREE_VARS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY_FREE_VARS); + /* Copy closure variables to free variables */ + PyCodeObject *co = _PyFrame_GetCode(frame); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; + assert(oparg == co->co_nfreevars); + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_DELETE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_DELETE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_ATTR); + _PyStackRef owner; + owner = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(owner); + if (err) goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_DELETE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_DELETE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_DEREF); + PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + // Can't use ERROR_IF here. + // Fortunately we don't need its superpower. + PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); + if (oldobj == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + Py_DECREF(oldobj); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_DELETE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_DELETE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_FAST); + _PyStackRef v = GETLOCAL(oparg); + if (PyStackRef_IsNull(v)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + SETLOCAL(oparg, PyStackRef_NULL); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_DELETE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_DELETE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_GLOBAL); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_Pop(GLOBALS(), name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Can't use ERROR_IF here. + if (err < 0) { + goto error; + } + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_DELETE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_DELETE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_NAME); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals when deleting %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_DelItem(ns, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Can't use ERROR_IF here. + if (err != 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, + name); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_DELETE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_DELETE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_SUBSCR); + _PyStackRef container; + _PyStackRef sub; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + /* del container[sub] */ + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), + PyStackRef_AsPyObjectBorrow(sub)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (err) goto pop_2_error; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_DICT_MERGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_DICT_MERGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_MERGE); + _PyStackRef callable; + _PyStackRef dict; + _PyStackRef update; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_MergeEx(dict_o, update_o, 2); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatKwargsError(tstate, callable_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(update); + goto pop_1_error; + } + PyStackRef_CLOSE(update); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_DICT_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_DICT_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_UPDATE); + _PyStackRef dict; + _PyStackRef update; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_Update(dict_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object is not a mapping", + Py_TYPE(update_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + PyStackRef_CLOSE(update); + goto pop_1_error; + } + PyStackRef_CLOSE(update); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_END_ASYNC_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_END_ASYNC_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(END_ASYNC_FOR); + _PyStackRef awaitable_st; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; + awaitable_st = stack_pointer[-2]; + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(exc && PyExceptionInstance_Check(exc)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + PyStackRef_CLOSE(awaitable_st); + PyStackRef_CLOSE(exc_st); + } + else { + Py_INCREF(exc); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto exception_unwind; + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_FOR); + _PyStackRef value; + value = stack_pointer[-1]; + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_SEND); + _PyStackRef receiver; + _PyStackRef value; + _PyStackRef val; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + (void)receiver; + val = value; + PyStackRef_CLOSE(receiver); + stack_pointer[-2] = val; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_ENTER_EXECUTOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_ENTER_EXECUTOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(ENTER_EXECUTOR); + #ifdef _Py_TIER2 + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; + assert(executor->vm_data.index == INSTR_OFFSET() - 1); + assert(executor->vm_data.code == code); + assert(executor->vm_data.valid); + assert(tstate->previous_executor == NULL); + /* If the eval breaker is set then stay in tier 1. + * This avoids any potentially infinite loops + * involving _RESUME_CHECK */ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + opcode = executor->vm_data.opcode; + oparg = (oparg & ~255) | executor->vm_data.oparg; + next_instr = this_instr; + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + DISPATCH_GOTO(); + } + tstate->previous_executor = Py_None; + Py_INCREF(executor); + GOTO_TIER_TWO(executor); + #else + Py_FatalError("ENTER_EXECUTOR is not supported in this build"); + #endif /* _Py_TIER2 */ + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_EXIT_INIT_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_EXIT_INIT_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXIT_INIT_CHECK); + _PyStackRef should_be_none; + should_be_none = stack_pointer[-1]; + assert(STACK_LEVEL() == 2); + if (!PyStackRef_IsNone(should_be_none)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyErr_Format(PyExc_TypeError, + "__init__() should return None, not '%.200s'", + Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_EXTENDED_ARG(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_EXTENDED_ARG(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXTENDED_ARG); + assert(oparg); + opcode = next_instr->op.code; + oparg = oparg << 8 | next_instr->op.arg; + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_FORMAT_SIMPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_FORMAT_SIMPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_SIMPLE); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + /* If value is a unicode object, then we know the result + * of format(value) is value itself. */ + if (!PyUnicode_CheckExact(value_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(value_o, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) goto pop_1_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + else { + res = value; + } + stack_pointer[-1] = res; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_FORMAT_WITH_SPEC(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_FORMAT_WITH_SPEC(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_WITH_SPEC); + _PyStackRef value; + _PyStackRef fmt_spec; + _PyStackRef res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + PyStackRef_CLOSE(fmt_spec); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER); + PREDICTED(FOR_ITER); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef iter; + _PyStackRef next; + // _SPECIALIZE_FOR_ITER + { + iter = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ForIter(iter, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(FOR_ITER); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + // _FOR_ITER + { + /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_o == NULL) { + if (_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!matches) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + PyStackRef_CLOSE(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */ + JUMPBY(oparg + 2); + DISPATCH(); + } + next = PyStackRef_FromPyObjectSteal(next_o); + // Common case: no jump, leave it to the code generator + } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_FOR_ITER_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_FOR_ITER_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_GEN); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyInterpreterFrame *gen_frame; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); + } + // _FOR_ITER_GEN_FRAME + { + iter = stack_pointer[-1]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); + STAT_INC(FOR_ITER, hit); + gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, PyStackRef_None); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + gen_frame->previous = frame; + // oparg is the return offset from the next instruction. + frame->return_offset = (uint16_t)( 2 + oparg); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_FOR_ITER_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_FOR_ITER_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_LIST); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_LIST + { + iter = stack_pointer[-1]; + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER); + } + // _ITER_JUMP_LIST + { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); + STAT_INC(FOR_ITER, hit); + PyListObject *seq = it->it_seq; + if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { + it->it_index = -1; + #ifndef Py_GIL_DISABLED + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); + } + #endif + PyStackRef_CLOSE(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ + JUMPBY(oparg + 2); + DISPATCH(); + } + } + // _ITER_NEXT_LIST + { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); + PyListObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyList_GET_SIZE(seq)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_FOR_ITER_RANGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_FOR_ITER_RANGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_RANGE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_RANGE + { + iter = stack_pointer[-1]; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + } + // _ITER_JUMP_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + STAT_INC(FOR_ITER, hit); + if (r->len <= 0) { + STACK_SHRINK(1); + PyStackRef_CLOSE(iter); + // Jump over END_FOR and POP_TOP instructions. + JUMPBY(oparg + 2); + DISPATCH(); + } + } + // _ITER_NEXT_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + assert(r->len > 0); + long value = r->start; + r->start = value + r->step; + r->len--; + PyObject *res = PyLong_FromLong(value); + if (res == NULL) goto error; + next = PyStackRef_FromPyObjectSteal(res); + } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_FOR_ITER_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_FOR_ITER_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_TUPLE + { + iter = stack_pointer[-1]; + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER); + } + // _ITER_JUMP_TUPLE + { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); + STAT_INC(FOR_ITER, hit); + PyTupleObject *seq = it->it_seq; + if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); + } + PyStackRef_CLOSE(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ + JUMPBY(oparg + 2); + DISPATCH(); + } + } + // _ITER_NEXT_TUPLE + { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); + PyTupleObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyTuple_GET_SIZE(seq)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); + } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_GET_AITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_GET_AITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AITER); + _PyStackRef obj; + _PyStackRef iter; + obj = stack_pointer[-1]; + unaryfunc getter = NULL; + PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); + PyObject *iter_o; + PyTypeObject *type = Py_TYPE(obj_o); + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; + } + if (getter == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an object with " + "__aiter__ method, got %.100s", + type->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(obj); + goto pop_1_error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + iter_o = (*getter)(obj_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(obj); + if (iter_o == NULL) goto pop_1_error; + if (Py_TYPE(iter_o)->tp_as_async == NULL || + Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' received an object from __aiter__ " + "that does not implement __anext__: %.100s", + Py_TYPE(iter_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(iter_o); + goto error; + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[-1] = iter; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_GET_ANEXT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_GET_ANEXT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ANEXT); + _PyStackRef aiter; + _PyStackRef awaitable; + aiter = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (awaitable_o == NULL) { + goto error; + } + awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); + stack_pointer[0] = awaitable; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_GET_AWAITABLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_GET_AWAITABLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AWAITABLE); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(iterable); + if (iter_o == NULL) goto pop_1_error; + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[-1] = iter; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_GET_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_GET_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ITER); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + /* before: [obj]; after [getiter(obj)] */ + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(iterable); + if (iter_o == NULL) goto pop_1_error; + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[-1] = iter; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_GET_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_GET_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_LEN); + _PyStackRef obj; + _PyStackRef len; + obj = stack_pointer[-1]; + // PUSH(len(TOS)) + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (len_i < 0) goto error; + PyObject *len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) goto error; + len = PyStackRef_FromPyObjectSteal(len_o); + stack_pointer[0] = len; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_GET_YIELD_FROM_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_GET_YIELD_FROM_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_YIELD_FROM_ITER); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + /* before: [obj]; after [getiter(obj)] */ + PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); + if (PyCoro_CheckExact(iterable_o)) { + /* `iterable` is a coroutine */ + if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + /* and it is used in a 'yield from' expression of a + regular generator. */ + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_TypeError, + "cannot 'yield from' a coroutine object " + "in a non-coroutine generator"); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + iter = iterable; + } + else { + if (PyGen_CheckExact(iterable_o)) { + iter = iterable; + } + else { + /* `iterable` is not a generator. */ + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(iterable_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + goto error; + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + PyStackRef_CLOSE(iterable); + } + } + stack_pointer[-1] = iter; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_IMPORT_FROM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_IMPORT_FROM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_FROM); + _PyStackRef from; + _PyStackRef res; + from = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) goto error; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_IMPORT_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_IMPORT_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_NAME); + _PyStackRef level; + _PyStackRef fromlist; + _PyStackRef res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportName(tstate, frame, name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(level); + PyStackRef_CLOSE(fromlist); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef res; + /* Skip 3 cache entries */ + // _MAYBE_EXPAND_METHOD + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + func = &stack_pointer[-2 - oparg]; + maybe_self = &stack_pointer[-1 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); + } + } + // _MONITOR_CALL + { + args = &stack_pointer[-oparg]; + maybe_self = &stack_pointer[-1 - oparg]; + func = &stack_pointer[-2 - oparg]; + int is_meth = !PyStackRef_IsNull(maybe_self[0]); + PyObject *function = PyStackRef_AsPyObjectBorrow(func[0]); + PyObject *arg0; + if (is_meth) { + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self[0]); + } + else { + if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg0 = &_PyInstrumentation_MISSING; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg0 + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) goto error; + } + // _DO_CALL + { + self_or_null = maybe_self; + callable = func; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; + } + frame->return_offset = 4 ; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(res_o); + } + } + } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); + GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + uint32_t version = read_u32(&this_instr[2].cache); + (void)version; + int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2)); + int total_args = oparg + is_meth; + PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3)); + PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING + : PyStackRef_AsPyObjectBorrow(PEEK(total_args + 1)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) goto error; + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + GO_TO_INSTRUCTION(CALL_KW); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_FOR); + _PyStackRef receiver; + _PyStackRef value; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + /* Need to create a fake StopIteration error here, + * to conform to PEP 380 */ + if (PyStackRef_GenCheck(receiver)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } + } + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_SEND); + _PyStackRef receiver; + _PyStackRef value; + _PyStackRef val; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } + } + val = value; + PyStackRef_CLOSE(receiver); + stack_pointer[-2] = val; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); + /* Skip 1 cache entry */ + _PyStackRef iter_stackref = TOP(); + PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next != NULL) { + PUSH(PyStackRef_FromPyObjectSteal(next)); + } + else { + if (_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!matches) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + STACK_SHRINK(1); + PyStackRef_CLOSE(iter_stackref); + /* Skip END_FOR and POP_TOP */ + _Py_CODEUNIT *target = next_instr + oparg + 2; + INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_INSTRUCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_INSTRUCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + _PyFrame_SetStackPointer(frame, stack_pointer); + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_opcode < 0) goto error; + next_instr = this_instr; + if (_PyOpcode_Caches[next_opcode]) { + PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); + } + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + } + } + // _MONITOR_JUMP_BACKWARD + { + INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_LINE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_LINE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_LINE); + int original_opcode = 0; + if (tstate->tracing) { + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = code->_co_monitoring->lines[(int)(this_instr - _PyFrame_GetBytecode(frame))].original_opcode; + stack_pointer = _PyFrame_GetStackPointer(frame); + next_instr = this_instr; + } else { + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = _Py_call_instrumentation_line( + tstate, frame, this_instr, prev_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = this_instr+1; + goto error; + } + next_instr = frame->instr_ptr; + if (next_instr != this_instr) { + DISPATCH(); + } + } + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + /* Prevent the underlying instruction from specializing + * and overwriting the instrumentation. */ + PAUSE_ADAPTIVE_COUNTER(cache->counter); + } + opcode = original_opcode; + DISPATCH_GOTO(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); + /* Skip 1 cache entry */ + // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we + // don't want to specialize instrumented instructions + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); + /* Skip 1 cache entry */ + _PyStackRef cond = POP(); + assert(PyStackRef_BoolCheck(cond)); + int jump = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); + /* Skip 1 cache entry */ + _PyStackRef value_stackref = POP(); + int jump = PyStackRef_IsNone(value_stackref); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + else { + PyStackRef_CLOSE(value_stackref); + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); + /* Skip 1 cache entry */ + _PyStackRef value_stackref = POP(); + int jump = !PyStackRef_IsNone(value_stackref); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + PyStackRef_CLOSE(value_stackref); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); + /* Skip 1 cache entry */ + _PyStackRef cond = POP(); + assert(PyStackRef_BoolCheck(cond)); + int jump = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RESUME); + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (bytecode == NULL) goto error; + _PyFrame_SetStackPointer(frame, stack_pointer); + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + // Make sure this_instr gets reset correctley for any uops that + // follow + next_instr = frame->instr_ptr; + DISPATCH(); + } + #endif + } + // _MAYBE_INSTRUMENT + { + if (tstate->tracing == 0) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } + next_instr = this_instr; + DISPATCH(); + } + } + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + } + } + } + // _MONITOR_RESUME + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation( + tstate, oparg > 0, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) goto error; + if (frame->instr_ptr != this_instr) { + /* Instrumentation has jumped */ + next_instr = frame->instr_ptr; + } + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); + _PyStackRef val; + _PyStackRef retval; + _PyStackRef res; + // _RETURN_VALUE_EVENT + { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) goto error; + } + // _RETURN_VALUE + { + retval = val; + #if TIER_ONE + assert(frame != entry_frame); + #endif + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(frame->return_offset); + res = temp; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + _PyStackRef val; + _PyStackRef retval; + _PyStackRef value; + // _YIELD_VALUE_EVENT + { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } + if (frame->instr_ptr != this_instr) { + next_instr = frame->instr_ptr; + DISPATCH(); + } + } + // _YIELD_VALUE + { + retval = val; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + #if TIER_ONE + assert(frame != entry_frame); + #endif + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + value = temp; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INTERPRETER_EXIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INTERPRETER_EXIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INTERPRETER_EXIT); + _PyStackRef retval; + retval = stack_pointer[-1]; + assert(frame == entry_frame); + assert(_PyFrame_IsIncomplete(frame)); + /* Restore previous frame and return. */ + tstate->current_frame = frame->previous; + assert(!_PyErr_Occurred(tstate)); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + PyObject *result = PyStackRef_AsPyObjectSteal(retval); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + /* Not strictly necessary, but prevents warnings */ + return result; + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_IS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_IS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IS_OP); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD); + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + } + } + // _JUMP_BACKWARD + { + uint16_t the_counter = read_u16(&this_instr[1].cache); + (void)the_counter; + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + #ifdef _Py_TIER2 + #if ENABLE_SPECIALIZATION + _Py_BackoffCounter counter = this_instr[1].counter; + if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) { + _Py_CODEUNIT *start = this_instr; + /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ + while (oparg > 255) { + oparg >>= 8; + start--; + } + _PyExecutorObject *executor; + _PyFrame_SetStackPointer(frame, stack_pointer); + int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (optimized <= 0) { + this_instr[1].counter = restart_backoff_counter(counter); + if (optimized < 0) goto error; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + this_instr[1].counter = initial_jump_backoff_counter(); + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(tstate->previous_executor == NULL); + tstate->previous_executor = Py_None; + GOTO_TIER_TWO(executor); + } + } + else { + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + #endif /* ENABLE_SPECIALIZATION */ + #endif /* _Py_TIER2 */ + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + JUMPBY(-oparg); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_FORWARD); + JUMPBY(oparg); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_APPEND); + _PyStackRef list; + _PyStackRef v; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)); + if (err < 0) goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LIST_EXTEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LIST_EXTEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_EXTEND); + _PyStackRef list_st; + _PyStackRef iterable_st; + iterable_st = stack_pointer[-1]; + list_st = stack_pointer[-2 - (oparg-1)]; + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (none_val == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + PyStackRef_CLOSE(iterable_st); + goto pop_1_error; + } + assert(Py_IsNone(none_val)); + PyStackRef_CLOSE(iterable_st); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR); + PREDICTED(LOAD_ATTR); + _Py_CODEUNIT* const this_instr = next_instr - 10; + (void)this_instr; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self_or_null = PyStackRef_NULL; + // _SPECIALIZE_LOAD_ATTR + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 8 cache entries */ + // _LOAD_ATTR + { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + PyObject *attr_o; + if (oparg & 1) { + /* Designed to work in tandem with CALL, pushes two values. */ + attr_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (is_meth) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr_o != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + } + else { + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + meth | NULL | arg1 | ... | argN + */ + PyStackRef_CLOSE(owner); + if (attr_o == NULL) goto pop_1_error; + self_or_null = PyStackRef_NULL; + } + } + else { + /* Classic, pushes one value. */ + _PyFrame_SetStackPointer(frame, stack_pointer); + attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(owner); + if (attr_o == NULL) goto pop_1_error; + /* We need to define self_or_null on all paths */ + self_or_null = PyStackRef_NULL; + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + assert(type_version != 0); + DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + assert(type_version != 0); + DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); + } + // _GUARD_TYPE_VERSION + { + uint32_t type_version = read_u32(&this_instr[4].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + /* Skip 1 cache entry */ + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *getattribute = read_obj(&this_instr[6].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert((oparg & 1) == 0); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + PyTypeObject *cls = Py_TYPE(owner_o); + assert(type_version != 0); + DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)getattribute; + assert(func_version != 0); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 2); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( + tstate, PyStackRef_FromPyObjectNew(f), 2, frame); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). + STACK_SHRINK(1); + new_frame->localsplus[0] = owner; + new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); + frame->return_offset = 10 ; + DISPATCH_INLINED(new_frame); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _CHECK_MANAGED_OBJECT_HAS_VALUES + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + } + // _LOAD_ATTR_INSTANCE_VALUE + { + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = *value_ptr; + DEOPT_IF(attr_o == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr_o); + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectSteal(attr_o); + PyStackRef_CLOSE(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _CHECK_ATTR_METHOD_LAZY_DICT + { + uint16_t dictoffset = read_u16(&this_instr[4].cache); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = *(PyObject **)ptr; + /* This object has a __dict__, just not yet created */ + DEOPT_IF(dict != NULL, LOAD_ATTR); + } + /* Skip 1 cache entry */ + // _LOAD_ATTR_METHOD_LAZY_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + } + // _LOAD_ATTR_METHOD_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + /* Cached method object */ + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + PyDictKeysObject *mod_keys; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_MODULE_PUSH_KEYS + { + owner = stack_pointer[-1]; + uint32_t dict_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; + assert(dict != NULL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR); + mod_keys = keys; + } + // _LOAD_ATTR_MODULE_FROM_KEYS + { + uint16_t index = read_u16(&this_instr[4].cache); + assert(mod_keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries)); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); + // Clear mod_keys from stack in case we need to deopt + DEOPT_IF(attr_o == NULL, LOAD_ATTR); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + DEOPT_IF(true, LOAD_ATTR); + } + #else + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + PyStackRef_CLOSE(owner); + attr = PyStackRef_FromPyObjectNew(descr); + } + stack_pointer[-1] = attr; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + } + // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + PyStackRef_CLOSE(owner); + attr = PyStackRef_FromPyObjectNew(descr); + } + stack_pointer[-1] = attr; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_PROPERTY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_PROPERTY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + } + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_PROPERTY_FRAME + { + PyObject *fget = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_IS_TYPE(fget, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)fget; + PyCodeObject *code = (PyCodeObject *)f->func_code; + DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR); + DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR); + DEOPT_IF(code->co_argcount != 1, LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); + new_frame->localsplus[0] = owner; + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _LOAD_ATTR_SLOT + { + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + char *addr = (char *)owner_o + index; + PyObject *attr_o = *(PyObject **)addr; + DEOPT_IF(attr_o == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectNew(attr_o); + PyStackRef_CLOSE(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _CHECK_ATTR_WITH_HINT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + DEOPT_IF(dict == NULL, LOAD_ATTR); + assert(PyDict_CheckExact((PyObject *)dict)); + } + // _LOAD_ATTR_WITH_HINT + { + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject *attr_o; + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), LOAD_ATTR); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, LOAD_ATTR); + attr_o = ep->me_value; + DEOPT_IF(attr_o == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_BUILD_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_BUILD_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_BUILD_CLASS); + _PyStackRef bc; + PyObject *bc_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) goto error; + if (bc_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + bc = PyStackRef_FromPyObjectSteal(bc_o); + stack_pointer[0] = bc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_COMMON_CONSTANT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_COMMON_CONSTANT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); + _PyStackRef value; + // Keep in sync with _common_constants in opcode.py + // If we ever have more than two constants, use a lookup table + PyObject *val; + if (oparg == CONSTANT_ASSERTIONERROR) { + val = PyExc_AssertionError; + } + else { + assert(oparg == CONSTANT_NOTIMPLEMENTEDERROR); + val = PyExc_NotImplementedError; + } + value = PyStackRef_FromPyObjectImmortal(val); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_CONST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_CONST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST); + PREDICTED(LOAD_CONST); + _PyStackRef value; + value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_CONST_IMMORTAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_CONST_IMMORTAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST_IMMORTAL); + static_assert(0 == 0, "incorrect cache size"); + _PyStackRef value; + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + assert(_Py_IsImmortal(obj)); + value = PyStackRef_FromPyObjectImmortal(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_DEREF); + _PyStackRef value; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *value_o = PyCell_GetRef(cell); + if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + value = PyStackRef_FromPyObjectSteal(value_o); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST); + _PyStackRef value; + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_FAST_AND_CLEAR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_FAST_AND_CLEAR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); + _PyStackRef value; + value = GETLOCAL(oparg); + // do not use SETLOCAL here, it decrefs the old value + GETLOCAL(oparg) = PyStackRef_NULL; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_FAST_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_FAST_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_CHECK); + _PyStackRef value; + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + value = PyStackRef_DUP(value_s); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); + _PyStackRef value1; + _PyStackRef value2; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = PyStackRef_DUP(GETLOCAL(oparg1)); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); + _PyStackRef class_dict_st; + _PyStackRef value; + class_dict_st = stack_pointer[-1]; + PyObject *value_o; + PyObject *name; + PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); + assert(class_dict); + assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + goto error; + } + if (!value_o) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + value_o = PyCell_GetRef(cell); + if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + } + PyStackRef_CLOSE(class_dict_st); + value = PyStackRef_FromPyObjectSteal(value_o); + stack_pointer[-1] = value; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); + _PyStackRef mod_or_class_dict; + _PyStackRef v; + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *v_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(mod_or_class_dict); + if (err < 0) goto pop_1_error; + if (v_o == NULL) { + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + goto error; + } + } + else { + /* Slow-path if globals or builtins is not a dict */ + /* namespace 1: globals */ + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) goto error; + if (v_o == NULL) { + /* namespace 2: builtins */ + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) goto error; + if (v_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + } + } + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[-1] = v; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL); + PREDICTED(LOAD_GLOBAL); + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + _PyStackRef *res; + _PyStackRef null = PyStackRef_NULL; + // _SPECIALIZE_LOAD_GLOBAL + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_GLOBAL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + // _LOAD_GLOBAL + { + res = &stack_pointer[0]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(*res)) goto error; + null = PyStackRef_NULL; + } + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_GLOBAL_BUILTIN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_GLOBAL_BUILTIN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + PyDictKeysObject *builtins_keys; + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION + { + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(keys)); + } + // _GUARD_BUILTINS_VERSION_PUSH_KEYS + { + uint16_t version = read_u16(&this_instr[3].cache); + PyDictObject *dict = (PyDictObject *)BUILTINS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + builtins_keys = keys; + assert(DK_IS_UNICODE(builtins_keys)); + } + // _LOAD_GLOBAL_BUILTINS_FROM_KEYS + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + DEOPT_IF(!increfed, LOAD_GLOBAL); + #else + Py_INCREF(res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + null = PyStackRef_NULL; + } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_GLOBAL_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_GLOBAL_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + PyDictKeysObject *globals_keys; + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION_PUSH_KEYS + { + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + globals_keys = keys; + assert(DK_IS_UNICODE(globals_keys)); + } + /* Skip 1 cache entry */ + // _LOAD_GLOBAL_MODULE_FROM_KEYS + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + DEOPT_IF(!increfed, LOAD_GLOBAL); + #else + Py_INCREF(res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + null = PyStackRef_NULL; + } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_LOCALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_LOCALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_LOCALS); + _PyStackRef locals; + PyObject *l = LOCALS(); + if (l == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + locals = PyStackRef_FromPyObjectNew(l); + stack_pointer[0] = locals; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_NAME); + _PyStackRef v; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) goto error; + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_SMALL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_SMALL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SMALL_INT); + _PyStackRef value; + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectImmortal(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_SPECIAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_SPECIAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SPECIAL); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self_or_null; + owner = stack_pointer[-1]; + assert(oparg <= SPECIAL_MAX); + PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); + PyObject *name = _Py_SpecialMethods[oparg].name; + PyObject *self_or_null_o; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = _PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + _Py_SpecialMethods[oparg].error, + Py_TYPE(owner_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + goto error; + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + self_or_null = self_or_null_o == NULL ? + PyStackRef_NULL : PyStackRef_FromPyObjectSteal(self_or_null_o); + stack_pointer[0] = attr; + stack_pointer[1] = self_or_null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR); + PREDICTED(LOAD_SUPER_ATTR); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + // _SPECIALIZE_LOAD_SUPER_ATTR + { + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + int load_method = oparg & 1; + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _LOAD_SUPER_ATTR + { + self_st = stack_pointer[-1]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + goto pop_3_error; + } + } + // we make no attempt to optimize here; specializations should + // handle any case whose performance we care about + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(super); + } + } + } + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + if (super == NULL) goto pop_3_error; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(super); + if (attr_o == NULL) goto error; + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; + } + stack_pointer[0] = attr; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr_st; + /* Skip 1 cache entry */ + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(!(oparg & 1)); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + if (attr == NULL) goto pop_3_error; + attr_st = PyStackRef_FromPyObjectSteal(attr); + stack_pointer[-3] = attr_st; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef self_or_null; + /* Skip 1 cache entry */ + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(oparg & 1); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyTypeObject *cls = (PyTypeObject *)class; + int method_found = 0; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + if (attr_o == NULL) { + PyStackRef_CLOSE(self_st); + goto pop_3_error; + } + if (method_found) { + self_or_null = self_st; // transfer ownership + } else { + PyStackRef_CLOSE(self_st); + self_or_null = PyStackRef_NULL; + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MAKE_CELL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MAKE_CELL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_CELL); + // "initial" is probably NULL but not if it's an arg (or set + // via the f_locals proxy before MAKE_CELL has run). + PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *cell = PyCell_New(initial); + if (cell == NULL) { + goto error; + } + SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell)); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MAKE_FUNCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MAKE_FUNCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_FUNCTION); + _PyStackRef codeobj_st; + _PyStackRef func; + codeobj_st = stack_pointer[-1]; + PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(codeobj_st); + if (func_obj == NULL) goto pop_1_error; + _PyFunction_SetVersion( + func_obj, ((PyCodeObject *)codeobj)->co_version); + func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); + stack_pointer[-1] = func; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MAP_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MAP_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAP_ADD); + _PyStackRef dict_st; + _PyStackRef key; + _PyStackRef value; + value = stack_pointer[-1]; + key = stack_pointer[-2]; + dict_st = stack_pointer[-3 - (oparg - 1)]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + /* dict[key] = value */ + // Do not DECREF INPUTS because the function steals the references + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2( + (PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(key), + PyStackRef_AsPyObjectSteal(value) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto pop_2_error; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MATCH_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MATCH_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_CLASS); + _PyStackRef subject; + _PyStackRef type; + _PyStackRef names; + _PyStackRef attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; + // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or + // None on failure. + assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attrs_o = _PyEval_MatchClass(tstate, + PyStackRef_AsPyObjectBorrow(subject), + PyStackRef_AsPyObjectBorrow(type), oparg, + PyStackRef_AsPyObjectBorrow(names)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(subject); + PyStackRef_CLOSE(type); + PyStackRef_CLOSE(names); + if (attrs_o) { + assert(PyTuple_CheckExact(attrs_o)); // Success! + attrs = PyStackRef_FromPyObjectSteal(attrs_o); + } + else { + if (_PyErr_Occurred(tstate)) goto pop_3_error; + // Error! + attrs = PyStackRef_None; // Failure! + } + stack_pointer[-3] = attrs; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MATCH_KEYS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MATCH_KEYS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_KEYS); + _PyStackRef subject; + _PyStackRef keys; + _PyStackRef values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; + // On successful match, PUSH(values). Otherwise, PUSH(None). + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, + PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (values_or_none_o == NULL) goto error; + values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); + stack_pointer[0] = values_or_none; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MATCH_MAPPING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MATCH_MAPPING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_MAPPING); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MATCH_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MATCH_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_SEQUENCE); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_NOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_NOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOP); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOT_TAKEN); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_POP_EXCEPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_POP_EXCEPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_EXCEPT); + _PyStackRef exc_value; + exc_value = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XSETREF(exc_info->exc_value, + PyStackRef_IsNone(exc_value) + ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_FALSE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NONE); + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + { + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; + } + else { + b = PyStackRef_False; + PyStackRef_CLOSE(value); + } + } + // _POP_JUMP_IF_TRUE + { + cond = b; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + { + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; + } + else { + b = PyStackRef_False; + PyStackRef_CLOSE(value); + } + } + // _POP_JUMP_IF_FALSE + { + cond = b; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_TRUE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_POP_TOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_POP_TOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_TOP); + _PyStackRef value; + value = stack_pointer[-1]; + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_PUSH_EXC_INFO(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_PUSH_EXC_INFO(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_EXC_INFO); + _PyStackRef exc; + _PyStackRef prev_exc; + _PyStackRef new_exc; + exc = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); + } + else { + prev_exc = PyStackRef_None; + } + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_PUSH_NULL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_PUSH_NULL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_NULL); + _PyStackRef res; + res = PyStackRef_NULL; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_RAISE_VARARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_RAISE_VARARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(RAISE_VARARGS); + _PyStackRef *args; + args = &stack_pointer[-oparg]; + assert(oparg < 3); + PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; + PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = do_raise(tstate, exc, cause); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + assert(oparg == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto exception_unwind; + } + goto error; + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_RERAISE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_RERAISE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(RERAISE); + _PyStackRef *values; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); + assert(oparg >= 0 && oparg <= 2); + if (oparg) { + PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]); + if (PyLong_Check(lasti)) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyLong_AsLong(lasti); + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!_PyErr_Occurred(tstate)); + } + else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(exc); + goto error; + } + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } + assert(exc && PyExceptionInstance_Check(exc)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto exception_unwind; + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_RESERVED(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_RESERVED(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESERVED); + assert(0 && "Executing RESERVED instruction."); + Py_FatalError("Executing RESERVED instruction."); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME); + PREDICTED(RESUME); + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (bytecode == NULL) goto error; + _PyFrame_SetStackPointer(frame, stack_pointer); + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + // Make sure this_instr gets reset correctley for any uops that + // follow + next_instr = frame->instr_ptr; + DISPATCH(); + } + #endif + } + // _MAYBE_INSTRUMENT + { + if (tstate->tracing == 0) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } + next_instr = this_instr; + DISPATCH(); + } + } + } + // _QUICKEN_RESUME + { + #if ENABLE_SPECIALIZATION_FT + if (tstate->tracing == 0 && this_instr->op.code == RESUME) { + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + } + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + } + } + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_RESUME_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_RESUME_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME_CHECK); + static_assert(0 == 0, "incorrect cache size"); + #if defined(__EMSCRIPTEN__) + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + DEOPT_IF(eval_breaker != version, RESUME); + #ifdef Py_GIL_DISABLED + DEOPT_IF(frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME); + #endif + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_RETURN_GENERATOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_RETURN_GENERATOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_GENERATOR); + _PyStackRef res; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (gen == NULL) goto error; + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectSteal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_VALUE); + _PyStackRef retval; + _PyStackRef res; + retval = stack_pointer[-1]; + #if TIER_ONE + assert(frame != entry_frame); + #endif + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(frame->return_offset); + res = temp; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND); + PREDICTED(SEND); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef receiver; + _PyStackRef v; + _PyStackRef retval; + // _SPECIALIZE_SEND + { + receiver = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Send(receiver, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(SEND); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _SEND + { + v = stack_pointer[-1]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + PyObject *retval_o; + assert(frame != entry_frame); + if ((tstate->interp->eval_frame == NULL) && + (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && + ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver_o; + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2 + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2 + oparg); + assert(gen_frame->previous == NULL); + gen_frame->previous = frame; + DISPATCH_INLINED(gen_frame); + } + if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + retval_o = PyObject_CallMethodOneArg(receiver_o, + &_Py_ID(send), + PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + if (retval_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyGen_FetchStopIterationValue(&retval_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err == 0) { + assert(retval_o != NULL); + JUMPBY(oparg); + } + else { + PyStackRef_CLOSE(v); + goto pop_1_error; + } + } + PyStackRef_CLOSE(v); + retval = PyStackRef_FromPyObjectSteal(retval_o); + } + stack_pointer[-1] = retval; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SEND_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SEND_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_GEN); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef receiver; + _PyStackRef v; + _PyInterpreterFrame *gen_frame; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, SEND); + } + // _SEND_GEN_FRAME + { + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + STAT_INC(SEND, hit); + gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2 + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2 + oparg); + gen_frame->previous = frame; + } + // _PUSH_FRAME + { + new_frame = gen_frame; + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SETUP_ANNOTATIONS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SETUP_ANNOTATIONS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SETUP_ANNOTATIONS); + PyObject *ann_dict; + if (LOCALS() == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when setting up annotations"); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + /* check if __annotations__ in locals()... */ + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) goto error; + if (ann_dict == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + ann_dict = PyDict_New(); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (ann_dict == NULL) goto error; + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(ann_dict); + if (err) goto error; + } + else { + Py_DECREF(ann_dict); + } + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SET_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SET_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_ADD); + _PyStackRef set; + _PyStackRef v; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + if (err) goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); + _PyStackRef attr_st; + _PyStackRef func_in; + _PyStackRef func_out; + func_in = stack_pointer[-1]; + attr_st = stack_pointer[-2]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); + PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); + func_out = func_in; + assert(PyFunction_Check(func)); + size_t offset = _Py_FunctionAttributeOffsets[oparg]; + assert(offset != 0); + PyObject **ptr = (PyObject **)(((char *)func) + offset); + assert(*ptr == NULL); + *ptr = attr; + stack_pointer[-2] = func_out; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SET_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SET_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_UPDATE); + _PyStackRef set; + _PyStackRef iterable; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(iterable); + if (err < 0) goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR); + PREDICTED(STORE_ATTR); + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + _PyStackRef owner; + _PyStackRef v; + // _SPECIALIZE_STORE_ATTR + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_StoreAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(STORE_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 3 cache entries */ + // _STORE_ATTR + { + v = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(owner); + if (err) goto pop_2_error; + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION_AND_LOCK + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + DEOPT_IF(true, STORE_ATTR); + } + } + // _GUARD_DORV_NO_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + DEOPT_IF(true, STORE_ATTR); + } + } + // _STORE_ATTR_INSTANCE_VALUE + { + value = stack_pointer[-2]; + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + STAT_INC(STORE_ATTR, hit); + assert(_PyObject_GetManagedDict(owner_o) == NULL); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *old_value = *value_ptr; + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); + if (old_value == NULL) { + PyDictValues *values = _PyObject_InlineValues(owner_o); + Py_ssize_t index = value_ptr - values->values; + _PyDictValues_AddToInsertionOrder(values, index); + } + UNLOCK_OBJECT(owner_o); + Py_XDECREF(old_value); + PyStackRef_CLOSE(owner); + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + } + // _STORE_ATTR_SLOT + { + value = stack_pointer[-2]; + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + char *addr = (char *)owner_o + index; + STAT_INC(STORE_ATTR, hit); + PyObject *old_value = *(PyObject **)addr; + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); + Py_XDECREF(old_value); + PyStackRef_CLOSE(owner); + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + } + // _STORE_ATTR_WITH_HINT + { + value = stack_pointer[-2]; + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + DEOPT_IF(dict == NULL, STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR); + #ifdef Py_GIL_DISABLED + if (dict != _PyObject_GetManagedDict(owner_o)) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } + #endif + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (hint >= (size_t)dict->ma_keys->dk_nentries || + !DK_IS_UNICODE(dict->ma_keys)) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } + PyObject *old_value = ep->me_value; + if (old_value == NULL) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(dict); + // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, + // when dict only holds the strong reference to value in ep->me_value. + Py_XDECREF(old_value); + STAT_INC(STORE_ATTR, hit); + PyStackRef_CLOSE(owner); + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_DEREF); + _PyStackRef v; + v = stack_pointer[-1]; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST); + _PyStackRef value; + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); + _PyStackRef value1; + _PyStackRef value2; + value1 = stack_pointer[-1]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[-1] = value2; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_FAST_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_FAST_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_STORE_FAST); + _PyStackRef value2; + _PyStackRef value1; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + SETLOCAL(oparg2, value2); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_GLOBAL); + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + if (err) goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_NAME); + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when storing %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + goto pop_1_error; + } + if (PyDict_CheckExact(ns)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + PyStackRef_CLOSE(v); + if (err) goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_SLICE); + _PyStackRef v; + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + // _SPECIALIZE_STORE_SLICE + { + // Placeholder until we implement STORE_SLICE specialization + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(STORE_SLICE); + #endif /* ENABLE_SPECIALIZATION */ + } + // _STORE_SLICE + { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); + int err; + if (slice == NULL) { + err = 1; + } + else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(slice); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); + if (err) goto pop_4_error; + } + stack_pointer += -4; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR); + PREDICTED(STORE_SUBSCR); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef container; + _PyStackRef sub; + _PyStackRef v; + // _SPECIALIZE_STORE_SUBSCR + { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_StoreSubscr(container, sub, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(STORE_SUBSCR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _STORE_SUBSCR + { + v = stack_pointer[-3]; + /* container[sub] = v */ + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (err) goto pop_3_error; + } + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef dict_st; + _PyStackRef sub; + /* Skip 1 cache entry */ + sub = stack_pointer[-1]; + dict_st = stack_pointer[-2]; + value = stack_pointer[-3]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + STAT_INC(STORE_SUBSCR, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(sub), + PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(dict_st); + if (err) goto pop_3_error; + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef list_st; + _PyStackRef sub_st; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + value = stack_pointer[-3]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + // Ensure nonnegative, zero-or-one-digit ints. + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR); + // Ensure index < len(list) + if (index >= PyList_GET_SIZE(list)) { + UNLOCK_OBJECT(list); + DEOPT_IF(true, STORE_SUBSCR); + } + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); + assert(old_value != NULL); + UNLOCK_OBJECT(list); // unlock before decrefs! + Py_DECREF(old_value); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(list_st); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SWAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SWAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SWAP); + _PyStackRef bottom_in; + _PyStackRef top_in; + _PyStackRef top_out; + _PyStackRef bottom_out; + top_in = stack_pointer[-1]; + bottom_in = stack_pointer[-2 - (oparg-2)]; + bottom_out = bottom_in; + top_out = top_in; + assert(oparg >= 2); + stack_pointer[-2 - (oparg-2)] = top_out; + stack_pointer[-1] = bottom_out; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_TO_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_TO_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL); + PREDICTED(TO_BOOL); + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + _PyStackRef value; + _PyStackRef res; + // _SPECIALIZE_TO_BOOL + { + value = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ToBool(value, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(TO_BOOL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _TO_BOOL + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (err < 0) goto pop_1_error; + res = err ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-1] = res; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL); + } + // _REPLACE_WITH_TRUE + { + value = owner; + PyStackRef_CLOSE(value); + res = PyStackRef_True; + } + stack_pointer[-1] = res; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_TO_BOOL_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_TO_BOOL_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_BOOL); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_TO_BOOL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_TO_BOOL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_INT); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL); + STAT_INC(TO_BOOL, hit); + if (_PyLong_IsZero((PyLongObject *)value_o)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; + } + else { + PyStackRef_CLOSE(value); + res = PyStackRef_True; + } + stack_pointer[-1] = res; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_TO_BOOL_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_TO_BOOL_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_LIST); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL); + STAT_INC(TO_BOOL, hit); + res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; + PyStackRef_CLOSE(value); + stack_pointer[-1] = res; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_TO_BOOL_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_TO_BOOL_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_NONE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + // This one is a bit weird, because we expect *some* failures: + DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + res = PyStackRef_False; + stack_pointer[-1] = res; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_TO_BOOL_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_TO_BOOL_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_STR); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL); + STAT_INC(TO_BOOL, hit); + if (value_o == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; + } + else { + assert(Py_SIZE(value_o)); + PyStackRef_CLOSE(value); + res = PyStackRef_True; + } + stack_pointer[-1] = res; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_UNARY_INVERT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_UNARY_INVERT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_INVERT); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) goto pop_1_error; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-1] = res; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_UNARY_NEGATIVE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_UNARY_NEGATIVE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NEGATIVE); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) goto pop_1_error; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-1] = res; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_UNARY_NOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_UNARY_NOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NOT); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + stack_pointer[-1] = res; + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_UNPACK_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_UNPACK_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNPACK_EX); + _PyStackRef seq; + _PyStackRef *right; + seq = stack_pointer[-1]; + right = &stack_pointer[(oparg & 0xFF)]; + _PyStackRef *top = right + (oparg >> 8); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(seq); + if (res == 0) goto pop_1_error; + stack_pointer += (oparg & 0xFF) + (oparg >> 8); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_UNPACK_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_UNPACK_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE); + PREDICTED(UNPACK_SEQUENCE); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef seq; + _PyStackRef *output; + // _SPECIALIZE_UNPACK_SEQUENCE + { + seq = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + (void)seq; + (void)counter; + } + // _UNPACK_SEQUENCE + { + output = &stack_pointer[-1]; + _PyStackRef *top = output + oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(seq); + if (res == 0) goto pop_1_error; + } + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_UNPACK_SEQUENCE_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_UNPACK_SEQUENCE_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef seq; + _PyStackRef *values; + /* Skip 1 cache entry */ + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE); + if (PyList_GET_SIZE(seq_o) != oparg) { + UNLOCK_OBJECT(seq_o); + DEOPT_IF(true, UNPACK_SEQUENCE); + } + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyList_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + UNLOCK_OBJECT(seq_o); + PyStackRef_CLOSE(seq); + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef seq; + _PyStackRef *values; + /* Skip 1 cache entry */ + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + PyStackRef_CLOSE(seq); + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + /* Skip 1 cache entry */ + seq = stack_pointer[-1]; + assert(oparg == 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + PyStackRef_CLOSE(seq); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_WITH_EXCEPT_START(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_WITH_EXCEPT_START(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(WITH_EXCEPT_START); + _PyStackRef exit_func; + _PyStackRef exit_self; + _PyStackRef lasti; + _PyStackRef val; + _PyStackRef res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_self = stack_pointer[-4]; + exit_func = stack_pointer[-5]; + /* At the top of the stack are 4 values: + - val: TOP = exc_info() + - unused: SECOND = previous exception + - lasti: THIRD = lasti of exception in exc_info() + - exit_self: FOURTH = the context or NULL + - exit_func: FIFTH = the context.__exit__ function or context.__exit__ bound method + We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). + Then we push the __exit__ return value. + */ + PyObject *exc, *tb; + PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); + PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); + assert(val_o && PyExceptionInstance_Check(val_o)); + exc = PyExceptionInstance_Class(val_o); + tb = PyException_GetTraceback(val_o); + if (tb == NULL) { + tb = Py_None; + } + else { + Py_DECREF(tb); + } + assert(PyStackRef_LongCheck(lasti)); + (void)lasti; // Shut up compiler warning if asserts are off + PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + int has_self = !PyStackRef_IsNull(exit_self); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) goto error; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_YIELD_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_YIELD_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(YIELD_VALUE); + _PyStackRef retval; + _PyStackRef value; + retval = stack_pointer[-1]; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + #if TIER_ONE + assert(frame != entry_frame); + #endif + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + value = temp; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +} +static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { + [BINARY_OP] = _TAIL_CALL_BINARY_OP, + [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, + [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, + [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_BINARY_OP_ADD_UNICODE, + [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE, + [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT, + [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_BINARY_OP_MULTIPLY_INT, + [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT, + [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_BINARY_OP_SUBTRACT_INT, + [BINARY_SLICE] = _TAIL_CALL_BINARY_SLICE, + [BINARY_SUBSCR] = _TAIL_CALL_BINARY_SUBSCR, + [BINARY_SUBSCR_DICT] = _TAIL_CALL_BINARY_SUBSCR_DICT, + [BINARY_SUBSCR_GETITEM] = _TAIL_CALL_BINARY_SUBSCR_GETITEM, + [BINARY_SUBSCR_LIST_INT] = _TAIL_CALL_BINARY_SUBSCR_LIST_INT, + [BINARY_SUBSCR_STR_INT] = _TAIL_CALL_BINARY_SUBSCR_STR_INT, + [BINARY_SUBSCR_TUPLE_INT] = _TAIL_CALL_BINARY_SUBSCR_TUPLE_INT, + [BUILD_LIST] = _TAIL_CALL_BUILD_LIST, + [BUILD_MAP] = _TAIL_CALL_BUILD_MAP, + [BUILD_SET] = _TAIL_CALL_BUILD_SET, + [BUILD_SLICE] = _TAIL_CALL_BUILD_SLICE, + [BUILD_STRING] = _TAIL_CALL_BUILD_STRING, + [BUILD_TUPLE] = _TAIL_CALL_BUILD_TUPLE, + [CACHE] = _TAIL_CALL_CACHE, + [CALL] = _TAIL_CALL_CALL, + [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT, + [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS, + [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_CALL_BOUND_METHOD_GENERAL, + [CALL_BUILTIN_CLASS] = _TAIL_CALL_CALL_BUILTIN_CLASS, + [CALL_BUILTIN_FAST] = _TAIL_CALL_CALL_BUILTIN_FAST, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS, + [CALL_BUILTIN_O] = _TAIL_CALL_CALL_BUILTIN_O, + [CALL_FUNCTION_EX] = _TAIL_CALL_CALL_FUNCTION_EX, + [CALL_INTRINSIC_1] = _TAIL_CALL_CALL_INTRINSIC_1, + [CALL_INTRINSIC_2] = _TAIL_CALL_CALL_INTRINSIC_2, + [CALL_ISINSTANCE] = _TAIL_CALL_CALL_ISINSTANCE, + [CALL_KW] = _TAIL_CALL_CALL_KW, + [CALL_KW_BOUND_METHOD] = _TAIL_CALL_CALL_KW_BOUND_METHOD, + [CALL_KW_NON_PY] = _TAIL_CALL_CALL_KW_NON_PY, + [CALL_KW_PY] = _TAIL_CALL_CALL_KW_PY, + [CALL_LEN] = _TAIL_CALL_CALL_LEN, + [CALL_LIST_APPEND] = _TAIL_CALL_CALL_LIST_APPEND, + [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS, + [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O, + [CALL_NON_PY_GENERAL] = _TAIL_CALL_CALL_NON_PY_GENERAL, + [CALL_PY_EXACT_ARGS] = _TAIL_CALL_CALL_PY_EXACT_ARGS, + [CALL_PY_GENERAL] = _TAIL_CALL_CALL_PY_GENERAL, + [CALL_STR_1] = _TAIL_CALL_CALL_STR_1, + [CALL_TUPLE_1] = _TAIL_CALL_CALL_TUPLE_1, + [CALL_TYPE_1] = _TAIL_CALL_CALL_TYPE_1, + [CHECK_EG_MATCH] = _TAIL_CALL_CHECK_EG_MATCH, + [CHECK_EXC_MATCH] = _TAIL_CALL_CHECK_EXC_MATCH, + [CLEANUP_THROW] = _TAIL_CALL_CLEANUP_THROW, + [COMPARE_OP] = _TAIL_CALL_COMPARE_OP, + [COMPARE_OP_FLOAT] = _TAIL_CALL_COMPARE_OP_FLOAT, + [COMPARE_OP_INT] = _TAIL_CALL_COMPARE_OP_INT, + [COMPARE_OP_STR] = _TAIL_CALL_COMPARE_OP_STR, + [CONTAINS_OP] = _TAIL_CALL_CONTAINS_OP, + [CONTAINS_OP_DICT] = _TAIL_CALL_CONTAINS_OP_DICT, + [CONTAINS_OP_SET] = _TAIL_CALL_CONTAINS_OP_SET, + [CONVERT_VALUE] = _TAIL_CALL_CONVERT_VALUE, + [COPY] = _TAIL_CALL_COPY, + [COPY_FREE_VARS] = _TAIL_CALL_COPY_FREE_VARS, + [DELETE_ATTR] = _TAIL_CALL_DELETE_ATTR, + [DELETE_DEREF] = _TAIL_CALL_DELETE_DEREF, + [DELETE_FAST] = _TAIL_CALL_DELETE_FAST, + [DELETE_GLOBAL] = _TAIL_CALL_DELETE_GLOBAL, + [DELETE_NAME] = _TAIL_CALL_DELETE_NAME, + [DELETE_SUBSCR] = _TAIL_CALL_DELETE_SUBSCR, + [DICT_MERGE] = _TAIL_CALL_DICT_MERGE, + [DICT_UPDATE] = _TAIL_CALL_DICT_UPDATE, + [END_ASYNC_FOR] = _TAIL_CALL_END_ASYNC_FOR, + [END_FOR] = _TAIL_CALL_END_FOR, + [END_SEND] = _TAIL_CALL_END_SEND, + [ENTER_EXECUTOR] = _TAIL_CALL_ENTER_EXECUTOR, + [EXIT_INIT_CHECK] = _TAIL_CALL_EXIT_INIT_CHECK, + [EXTENDED_ARG] = _TAIL_CALL_EXTENDED_ARG, + [FORMAT_SIMPLE] = _TAIL_CALL_FORMAT_SIMPLE, + [FORMAT_WITH_SPEC] = _TAIL_CALL_FORMAT_WITH_SPEC, + [FOR_ITER] = _TAIL_CALL_FOR_ITER, + [FOR_ITER_GEN] = _TAIL_CALL_FOR_ITER_GEN, + [FOR_ITER_LIST] = _TAIL_CALL_FOR_ITER_LIST, + [FOR_ITER_RANGE] = _TAIL_CALL_FOR_ITER_RANGE, + [FOR_ITER_TUPLE] = _TAIL_CALL_FOR_ITER_TUPLE, + [GET_AITER] = _TAIL_CALL_GET_AITER, + [GET_ANEXT] = _TAIL_CALL_GET_ANEXT, + [GET_AWAITABLE] = _TAIL_CALL_GET_AWAITABLE, + [GET_ITER] = _TAIL_CALL_GET_ITER, + [GET_LEN] = _TAIL_CALL_GET_LEN, + [GET_YIELD_FROM_ITER] = _TAIL_CALL_GET_YIELD_FROM_ITER, + [IMPORT_FROM] = _TAIL_CALL_IMPORT_FROM, + [IMPORT_NAME] = _TAIL_CALL_IMPORT_NAME, + [INSTRUMENTED_CALL] = _TAIL_CALL_INSTRUMENTED_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_CALL_KW] = _TAIL_CALL_INSTRUMENTED_CALL_KW, + [INSTRUMENTED_END_FOR] = _TAIL_CALL_INSTRUMENTED_END_FOR, + [INSTRUMENTED_END_SEND] = _TAIL_CALL_INSTRUMENTED_END_SEND, + [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_INSTRUMENTED_FOR_ITER, + [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_INSTRUMENTED_INSTRUCTION, + [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD, + [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD, + [INSTRUMENTED_LINE] = _TAIL_CALL_INSTRUMENTED_LINE, + [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR, + [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_INSTRUMENTED_NOT_TAKEN, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE, + [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE, + [INSTRUMENTED_RESUME] = _TAIL_CALL_INSTRUMENTED_RESUME, + [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_INSTRUMENTED_RETURN_VALUE, + [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_INSTRUMENTED_YIELD_VALUE, + [INTERPRETER_EXIT] = _TAIL_CALL_INTERPRETER_EXIT, + [IS_OP] = _TAIL_CALL_IS_OP, + [JUMP_BACKWARD] = _TAIL_CALL_JUMP_BACKWARD, + [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT, + [JUMP_FORWARD] = _TAIL_CALL_JUMP_FORWARD, + [LIST_APPEND] = _TAIL_CALL_LIST_APPEND, + [LIST_EXTEND] = _TAIL_CALL_LIST_EXTEND, + [LOAD_ATTR] = _TAIL_CALL_LOAD_ATTR, + [LOAD_ATTR_CLASS] = _TAIL_CALL_LOAD_ATTR_CLASS, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, + [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE, + [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT, + [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT, + [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES, + [LOAD_ATTR_MODULE] = _TAIL_CALL_LOAD_ATTR_MODULE, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, + [LOAD_ATTR_PROPERTY] = _TAIL_CALL_LOAD_ATTR_PROPERTY, + [LOAD_ATTR_SLOT] = _TAIL_CALL_LOAD_ATTR_SLOT, + [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_LOAD_ATTR_WITH_HINT, + [LOAD_BUILD_CLASS] = _TAIL_CALL_LOAD_BUILD_CLASS, + [LOAD_COMMON_CONSTANT] = _TAIL_CALL_LOAD_COMMON_CONSTANT, + [LOAD_CONST] = _TAIL_CALL_LOAD_CONST, + [LOAD_CONST_IMMORTAL] = _TAIL_CALL_LOAD_CONST_IMMORTAL, + [LOAD_DEREF] = _TAIL_CALL_LOAD_DEREF, + [LOAD_FAST] = _TAIL_CALL_LOAD_FAST, + [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_LOAD_FAST_AND_CLEAR, + [LOAD_FAST_CHECK] = _TAIL_CALL_LOAD_FAST_CHECK, + [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_LOAD_FAST_LOAD_FAST, + [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF, + [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS, + [LOAD_GLOBAL] = _TAIL_CALL_LOAD_GLOBAL, + [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_LOAD_GLOBAL_BUILTIN, + [LOAD_GLOBAL_MODULE] = _TAIL_CALL_LOAD_GLOBAL_MODULE, + [LOAD_LOCALS] = _TAIL_CALL_LOAD_LOCALS, + [LOAD_NAME] = _TAIL_CALL_LOAD_NAME, + [LOAD_SMALL_INT] = _TAIL_CALL_LOAD_SMALL_INT, + [LOAD_SPECIAL] = _TAIL_CALL_LOAD_SPECIAL, + [LOAD_SUPER_ATTR] = _TAIL_CALL_LOAD_SUPER_ATTR, + [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_LOAD_SUPER_ATTR_ATTR, + [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_LOAD_SUPER_ATTR_METHOD, + [MAKE_CELL] = _TAIL_CALL_MAKE_CELL, + [MAKE_FUNCTION] = _TAIL_CALL_MAKE_FUNCTION, + [MAP_ADD] = _TAIL_CALL_MAP_ADD, + [MATCH_CLASS] = _TAIL_CALL_MATCH_CLASS, + [MATCH_KEYS] = _TAIL_CALL_MATCH_KEYS, + [MATCH_MAPPING] = _TAIL_CALL_MATCH_MAPPING, + [MATCH_SEQUENCE] = _TAIL_CALL_MATCH_SEQUENCE, + [NOP] = _TAIL_CALL_NOP, + [NOT_TAKEN] = _TAIL_CALL_NOT_TAKEN, + [POP_EXCEPT] = _TAIL_CALL_POP_EXCEPT, + [POP_JUMP_IF_FALSE] = _TAIL_CALL_POP_JUMP_IF_FALSE, + [POP_JUMP_IF_NONE] = _TAIL_CALL_POP_JUMP_IF_NONE, + [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_POP_JUMP_IF_NOT_NONE, + [POP_JUMP_IF_TRUE] = _TAIL_CALL_POP_JUMP_IF_TRUE, + [POP_TOP] = _TAIL_CALL_POP_TOP, + [PUSH_EXC_INFO] = _TAIL_CALL_PUSH_EXC_INFO, + [PUSH_NULL] = _TAIL_CALL_PUSH_NULL, + [RAISE_VARARGS] = _TAIL_CALL_RAISE_VARARGS, + [RERAISE] = _TAIL_CALL_RERAISE, + [RESERVED] = _TAIL_CALL_RESERVED, + [RESUME] = _TAIL_CALL_RESUME, + [RESUME_CHECK] = _TAIL_CALL_RESUME_CHECK, + [RETURN_GENERATOR] = _TAIL_CALL_RETURN_GENERATOR, + [RETURN_VALUE] = _TAIL_CALL_RETURN_VALUE, + [SEND] = _TAIL_CALL_SEND, + [SEND_GEN] = _TAIL_CALL_SEND_GEN, + [SETUP_ANNOTATIONS] = _TAIL_CALL_SETUP_ANNOTATIONS, + [SET_ADD] = _TAIL_CALL_SET_ADD, + [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_SET_FUNCTION_ATTRIBUTE, + [SET_UPDATE] = _TAIL_CALL_SET_UPDATE, + [STORE_ATTR] = _TAIL_CALL_STORE_ATTR, + [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE, + [STORE_ATTR_SLOT] = _TAIL_CALL_STORE_ATTR_SLOT, + [STORE_ATTR_WITH_HINT] = _TAIL_CALL_STORE_ATTR_WITH_HINT, + [STORE_DEREF] = _TAIL_CALL_STORE_DEREF, + [STORE_FAST] = _TAIL_CALL_STORE_FAST, + [STORE_FAST_LOAD_FAST] = _TAIL_CALL_STORE_FAST_LOAD_FAST, + [STORE_FAST_STORE_FAST] = _TAIL_CALL_STORE_FAST_STORE_FAST, + [STORE_GLOBAL] = _TAIL_CALL_STORE_GLOBAL, + [STORE_NAME] = _TAIL_CALL_STORE_NAME, + [STORE_SLICE] = _TAIL_CALL_STORE_SLICE, + [STORE_SUBSCR] = _TAIL_CALL_STORE_SUBSCR, + [STORE_SUBSCR_DICT] = _TAIL_CALL_STORE_SUBSCR_DICT, + [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_STORE_SUBSCR_LIST_INT, + [SWAP] = _TAIL_CALL_SWAP, + [TO_BOOL] = _TAIL_CALL_TO_BOOL, + [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TO_BOOL_ALWAYS_TRUE, + [TO_BOOL_BOOL] = _TAIL_CALL_TO_BOOL_BOOL, + [TO_BOOL_INT] = _TAIL_CALL_TO_BOOL_INT, + [TO_BOOL_LIST] = _TAIL_CALL_TO_BOOL_LIST, + [TO_BOOL_NONE] = _TAIL_CALL_TO_BOOL_NONE, + [TO_BOOL_STR] = _TAIL_CALL_TO_BOOL_STR, + [UNARY_INVERT] = _TAIL_CALL_UNARY_INVERT, + [UNARY_NEGATIVE] = _TAIL_CALL_UNARY_NEGATIVE, + [UNARY_NOT] = _TAIL_CALL_UNARY_NOT, + [UNPACK_EX] = _TAIL_CALL_UNPACK_EX, + [UNPACK_SEQUENCE] = _TAIL_CALL_UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_UNPACK_SEQUENCE_LIST, + [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TUPLE, + [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, + [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, + [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, +}; +#undef TIER_ONE diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index fcdd3bdacd0e7a..8763eb02b1d107 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -128,6 +128,39 @@ def uses_this(inst: Instruction) -> bool: return False +def write_single_inst(out: CWriter, emitter: Emitter, name: str, inst: Instruction): + needs_this = uses_this(inst) + unused_guard = "(void)this_instr;\n" if inst.family is None else "" + if inst.properties.needs_prev: + out.emit(f"_Py_CODEUNIT* const prev_instr = frame->instr_ptr;\n") + if needs_this and not inst.is_target: + out.emit(f"_Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;\n") + out.emit(unused_guard) + else: + out.emit(f"frame->instr_ptr = next_instr;\n") + out.emit(f"next_instr += {inst.size};\n") + out.emit(f"INSTRUCTION_STATS({name});\n") + if inst.is_target: + out.emit(f"PREDICTED({name});\n") + if needs_this: + out.emit(f"_Py_CODEUNIT* const this_instr = next_instr - {inst.size};\n") + out.emit(unused_guard) + if inst.family is not None: + out.emit( + f"static_assert({inst.family.size} == {inst.size-1}" + ', "incorrect cache size");\n' + ) + declare_variables(inst, out) + offset = 1 # The instruction itself + stack = Stack() + for part in inst.parts: + # Only emit braces if more than one uop + insert_braces = len([p for p in inst.parts if isinstance(p, Uop)]) > 1 + offset, stack = write_uop(part, emitter, offset, stack, inst, insert_braces) + out.start_line() + + stack.flush(out) + def generate_tier1( filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool ) -> None: @@ -144,39 +177,9 @@ def generate_tier1( emitter = Emitter(out) out.emit("\n") for name, inst in sorted(analysis.instructions.items()): - needs_this = uses_this(inst) out.emit("\n") out.emit(f"TARGET({name}) {{\n") - unused_guard = "(void)this_instr;\n" if inst.family is None else "" - if inst.properties.needs_prev: - out.emit(f"_Py_CODEUNIT* const prev_instr = frame->instr_ptr;\n") - if needs_this and not inst.is_target: - out.emit(f"_Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;\n") - out.emit(unused_guard) - else: - out.emit(f"frame->instr_ptr = next_instr;\n") - out.emit(f"next_instr += {inst.size};\n") - out.emit(f"INSTRUCTION_STATS({name});\n") - if inst.is_target: - out.emit(f"PREDICTED({name});\n") - if needs_this: - out.emit(f"_Py_CODEUNIT* const this_instr = next_instr - {inst.size};\n") - out.emit(unused_guard) - if inst.family is not None: - out.emit( - f"static_assert({inst.family.size} == {inst.size-1}" - ', "incorrect cache size");\n' - ) - declare_variables(inst, out) - offset = 1 # The instruction itself - stack = Stack() - for part in inst.parts: - # Only emit braces if more than one uop - insert_braces = len([p for p in inst.parts if isinstance(p, Uop)]) > 1 - offset, stack = write_uop(part, emitter, offset, stack, inst, insert_braces) - out.start_line() - - stack.flush(out) + write_single_inst(out, emitter, name, inst) if not inst.parts[-1].properties.always_exits: out.emit("DISPATCH();\n") out.start_line() diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py new file mode 100644 index 00000000000000..838871763f0c25 --- /dev/null +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -0,0 +1,235 @@ +"""Generate the main interpreter tail call. +Reads the instruction definitions from bytecodes.c. +Writes the cases to generated_cases.c.h, which is #included in ceval.c. +""" + +import argparse +import re + +from typing import TextIO + +from generators_common import ( + ROOT, + write_header, +) + +from tier1_generator import ( + write_single_inst +) + +DEFAULT_INPUT = ROOT / "Python/generated_cases.c.h" +DEFAULT_OUTPUT = ROOT / "Python/generated_cases_tail_call.c.h" + + +"""Generate the main interpreter tail calls. +Reads the instruction definitions from bytecodes.c. +Writes the cases to generated_cases.c.h, which is #included in ceval.c. +""" + +import argparse + +from analyzer import ( + Analysis, + Instruction, + Uop, + Part, + analyze_files, + Skip, + Flush, + analysis_error, + StackItem, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, + type_and_null, + Emitter, +) +from cwriter import CWriter +from typing import TextIO +from stack import Stack + + +DEFAULT_OUTPUT = ROOT / "Python/generated_cases_tail_call.c.h" + + +FOOTER = "#undef TIER_ONE\n" + + +def generate_tier1( + filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool +) -> None: + write_header(__file__, filenames, outfile) + outfile.write( + """ +#ifndef Py_TAIL_CALL_INTERP + #error "This file is for tail-calling interpreter only." +#endif +#define TIER_ONE 1 +""" + ) + out = CWriter(outfile, 0, lines) + out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256];\n"); + emitter = Emitter(out) + out.emit("\n") + for name, inst in sorted(analysis.instructions.items()): + out.emit("\n") + out.emit(f""" +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_{name}(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_{name}(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +""") + out.emit("{\n") + # out.emit(f'printf("{name}\\n");\n') + out.emit("int opcode = next_instr->op.code;\n") + out.emit("{\n") + write_single_inst(out, emitter, name, inst) + if not inst.parts[-1].properties.always_exits: + out.emit("DISPATCH();\n") + out.emit(""" +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + +""") + out.start_line() + out.emit("}\n") + out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256] = {\n") + for name in sorted(analysis.instructions.keys()): + out.emit(f"[{name}] = _TAIL_CALL_{name},\n") + out.emit("};\n") + outfile.write(FOOTER) + + +arg_parser = argparse.ArgumentParser( + description="Generate the code for the interpreter switch.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + +arg_parser.add_argument( + "-l", "--emit-line-directives", help="Emit #line directives", action="store_true" +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_tier1(args.input, data, outfile, args.emit_line_directives) From c120a98af04676447c3f139afa96ec9866870813 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 5 Jan 2025 10:10:37 +0800 Subject: [PATCH 003/110] fix release builds --- Python/ceval_macros.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index bd57ffea4133af..cb4fe349ed09d8 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -86,7 +86,7 @@ typedef PyObject* (*py_tail_call_funcptr)(_PyInterpreterFrame *, _PyStackRef *, #else #define DISPATCH_GOTO() do { \ __attribute__((musttail)) \ - return (INSTRUCTION_TABLE[opcode])(frame, stack_pointer, tstate, next_instr, entry_frame, oparg); \ + return (INSTRUCTION_TABLE[opcode])(frame, stack_pointer, tstate, next_instr, oparg, entry_frame); \ } while (0) #endif #elif USE_COMPUTED_GOTOS @@ -163,6 +163,7 @@ do { \ return (INSTRUCTION_TABLE[opcode])(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); \ } while (0) #else +#define DISPATCH_INLINED(NEW_FRAME) \ do { \ assert(tstate->interp->eval_frame == NULL); \ _PyFrame_SetStackPointer(frame, stack_pointer); \ @@ -176,7 +177,7 @@ do { \ stack_pointer = _PyFrame_GetStackPointer(frame); \ NEXTOPARG(); \ __attribute__((musttail)) \ - return (INSTRUCTION_TABLE[opcode])(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); \ + return (INSTRUCTION_TABLE[opcode])(frame, stack_pointer, tstate, next_instr, oparg, entry_frame); \ } while (0) #endif #else From 6f753da4ffcb0caac97cd71c8abcb57c14308bf5 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 5 Jan 2025 18:50:09 +0800 Subject: [PATCH 004/110] split out cold error parts --- Python/bytecodes.c | 16 +- Python/ceval_macros.h | 19 +- Python/generated_cases_tail_call.c.h | 38898 ++++------------ Tools/cases_generator/generators_common.py | 11 +- .../tier1_tail_call_generator.py | 205 +- 5 files changed, 8508 insertions(+), 30641 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index facf28ad0a831d..97d9430a96749b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1012,7 +1012,7 @@ dummy_func( if (err) { assert(oparg == 0); monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; + CEVAL_GOTO(exception_unwind); } ERROR_IF(true, error); } @@ -1282,7 +1282,7 @@ dummy_func( assert(exc && PyExceptionInstance_Check(exc)); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; + CEVAL_GOTO(exception_unwind); } tier1 inst(END_ASYNC_FOR, (awaitable_st, exc_st -- )) { @@ -1297,7 +1297,8 @@ dummy_func( Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; + INPUTS_DEAD(); + CEVAL_GOTO(exception_unwind); } } @@ -1315,7 +1316,10 @@ dummy_func( else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; + INPUTS_DEAD(); + none = PyStackRef_NULL; + value = PyStackRef_NULL; + CEVAL_GOTO(exception_unwind); } } @@ -4003,7 +4007,7 @@ dummy_func( PyObject *res_o = PyLong_FromSsize_t(len_i); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { - GOTO_ERROR(error); + CEVAL_GOTO(error); } PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(arg_stackref); @@ -4740,7 +4744,7 @@ dummy_func( tstate, frame, this_instr, prev_instr); if (original_opcode < 0) { next_instr = this_instr+1; - goto error; + CEVAL_GOTO(error); } next_instr = frame->instr_ptr; if (next_instr != this_instr) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index cb4fe349ed09d8..f0d2c59d07508c 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -83,20 +83,31 @@ typedef PyObject* (*py_tail_call_funcptr)(_PyInterpreterFrame *, _PyStackRef *, __attribute__((musttail)) \ return (INSTRUCTION_TABLE[opcode])(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); \ } while (0) +#define CEVAL_GOTO(name) do { \ + __attribute__((musttail)) \ + return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); \ +} while (0) #else #define DISPATCH_GOTO() do { \ __attribute__((musttail)) \ return (INSTRUCTION_TABLE[opcode])(frame, stack_pointer, tstate, next_instr, oparg, entry_frame); \ } while (0) +#define CEVAL_GOTO(name) do { \ + __attribute__((musttail)) \ + return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, next_instr, oparg, entry_frame); \ +} while (0) #endif #elif USE_COMPUTED_GOTOS # define TARGET(op) TARGET_##op: # define DISPATCH_GOTO() goto *opcode_targets[opcode] +# define CEVAL_GOTO(name) goto name; #else # define TARGET(op) case op: TARGET_##op: # define DISPATCH_GOTO() goto dispatch_opcode +# define CEVAL_GOTO(name) goto name; #endif + /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ #ifdef LLTRACE #define PRE_DISPATCH_GOTO() if (lltrace >= 5) { \ @@ -110,7 +121,7 @@ typedef PyObject* (*py_tail_call_funcptr)(_PyInterpreterFrame *, _PyStackRef *, do { \ lltrace = maybe_lltrace_resume_frame(frame, entry_frame, GLOBALS()); \ if (lltrace < 0) { \ - goto exit_unwind; \ + CEVAL_GOTO(exit_unwind); \ } \ } while (0) #else @@ -150,13 +161,13 @@ do { \ frame = tstate->current_frame = (NEW_FRAME); \ CALL_STAT_INC(inlined_py_calls); \ if (_Py_EnterRecursivePy(tstate)) {\ - goto exit_unwind;\ + CEVAL_GOTO(exit_unwind);\ } \ next_instr = frame->instr_ptr; \ stack_pointer = _PyFrame_GetStackPointer(frame); \ lltrace = maybe_lltrace_resume_frame(frame, entry_frame, GLOBALS()); \ if (lltrace < 0) { \ - goto exit_unwind; \ + CEVAL_GOTO(exit_unwind); \ } \ NEXTOPARG(); \ __attribute__((musttail)) \ @@ -441,7 +452,7 @@ do { \ stack_pointer = _PyFrame_GetStackPointer(frame); \ if (next_instr == NULL) { \ next_instr = (dest)+1; \ - goto error; \ + CEVAL_GOTO(error); \ } \ } \ } while (0); diff --git a/Python/generated_cases_tail_call.c.h b/Python/generated_cases_tail_call.c.h index 59d4bafd20a556..7597c2e9fd6936 100644 --- a/Python/generated_cases_tail_call.c.h +++ b/Python/generated_cases_tail_call.c.h @@ -9,151 +9,52 @@ #define TIER_ONE 1 static py_tail_call_funcptr INSTRUCTION_TABLE[256]; - - #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP); - PREDICTED(BINARY_OP); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef lhs; - _PyStackRef rhs; - _PyStackRef res; - // _SPECIALIZE_BINARY_OP - { - rhs = stack_pointer[-1]; - lhs = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(BINARY_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - assert(NB_ADD <= oparg); - assert(oparg <= NB_INPLACE_XOR); - } - // _BINARY_OP - { - PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); - PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); - assert(_PyEval_BinaryOps[oparg]); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(lhs); - PyStackRef_CLOSE(rhs); - if (res_o == NULL) goto pop_2_error; - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); +; -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_resume_with_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_resume_with_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif +{ + int opcode = next_instr->op.code; - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + #ifdef LLTRACE + __attribute__((musttail)) + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); + #else + __attribute__((musttail)) + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame); + #endif +} - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_exit_unwind(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_exit_unwind(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } - } +{ + int opcode = next_instr->op.code; -exit_unwind: assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); assert(frame != entry_frame); @@ -168,90 +69,27 @@ _TAIL_CALL_BINARY_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return NULL; } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - + #ifdef LLTRACE + __attribute__((musttail)) + return _TAIL_CALL_resume_with_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); + #else + __attribute__((musttail)) + return _TAIL_CALL_resume_with_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame); + #endif } - #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_OP_ADD_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_exception_unwind(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_OP_ADD_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_exception_unwind(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_FLOAT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); - } - /* Skip 1 cache entry */ - // _BINARY_OP_ADD_FLOAT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval + - ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) goto pop_2_error; - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: { /* We can't use frame->instr_ptr here, as RERAISE may have set it */ int offset = INSTR_OFFSET()-1; @@ -268,7 +106,7 @@ _TAIL_CALL_BINARY_OP_ADD_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_po assert(STACK_LEVEL() == 0); _PyFrame_SetStackPointer(frame, stack_pointer); monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; + CEVAL_GOTO(exit_unwind); } assert(STACK_LEVEL() >= level); @@ -280,7 +118,7 @@ _TAIL_CALL_BINARY_OP_ADD_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_po int frame_lasti = _PyInterpreterFrame_LASTI(frame); PyObject *lasti = PyLong_FromLong(frame_lasti); if (lasti == NULL) { - goto exception_unwind; + CEVAL_GOTO(exception_unwind); } PUSH(PyStackRef_FromPyObjectSteal(lasti)); } @@ -294,7 +132,7 @@ _TAIL_CALL_BINARY_OP_ADD_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_po next_instr = _PyFrame_GetBytecode(frame) + handler; if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; + CEVAL_GOTO(exception_unwind); } /* Resume normal execution */ #ifdef LLTRACE @@ -304,86 +142,20 @@ _TAIL_CALL_BINARY_OP_ADD_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_po #endif DISPATCH(); } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_OP_ADD_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_OP_ADD_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_ADD_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_INT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); - } - /* Skip 1 cache entry */ - // _BINARY_OP_ADD_INT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) goto pop_2_error; - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: /* Double-check exception status. */ #ifdef NDEBUG if (!_PyErr_Occurred(tstate)) { @@ -403,82 +175,257 @@ _TAIL_CALL_BINARY_OP_ADD_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_poin } } _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); + #ifdef LLTRACE + __attribute__((musttail)) + return _TAIL_CALL_exception_unwind(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); + #else + __attribute__((musttail)) + return _TAIL_CALL_exception_unwind(frame, stack_pointer, tstate, next_instr, oparg, entry_frame); + #endif +} - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_pop_1_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_pop_1_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + STACK_SHRINK(1); + #ifdef LLTRACE + __attribute__((musttail)) + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); + #else + __attribute__((musttail)) + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame); + #endif +} - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_pop_2_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_pop_2_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + STACK_SHRINK(1); + #ifdef LLTRACE + __attribute__((musttail)) + return _TAIL_CALL_pop_1_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); + #else + __attribute__((musttail)) + return _TAIL_CALL_pop_1_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame); + #endif +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_pop_3_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_pop_3_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + STACK_SHRINK(1); + #ifdef LLTRACE + __attribute__((musttail)) + return _TAIL_CALL_pop_2_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); + #else + __attribute__((musttail)) + return _TAIL_CALL_pop_2_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame); + #endif +} - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_pop_4_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_pop_4_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); +{ + int opcode = next_instr->op.code; + STACK_SHRINK(1); + #ifdef LLTRACE + __attribute__((musttail)) + return _TAIL_CALL_pop_3_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); + #else + __attribute__((musttail)) + return _TAIL_CALL_pop_3_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame); + #endif +} + + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP); + PREDICTED(BINARY_OP); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef lhs; + _PyStackRef rhs; + _PyStackRef res; + // _SPECIALIZE_BINARY_OP + { + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(BINARY_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + assert(NB_ADD <= oparg); + assert(oparg <= NB_INPLACE_XOR); } + // _BINARY_OP + { + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); + assert(_PyEval_BinaryOps[oparg]); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(lhs); + PyStackRef_CLOSE(rhs); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } +} -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP_ADD_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP_ADD_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_FLOAT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_ADD_FLOAT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } +} -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP_ADD_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP_ADD_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_INT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_ADD_INT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } } @@ -519,119 +466,15 @@ _TAIL_CALL_BINARY_OP_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef *stack_ PyObject *res_o = PyUnicode_Concat(left_o, right_o); PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) CEVAL_GOTO(pop_2_error); res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} + } +} #ifdef LLTRACE @@ -693,7 +536,7 @@ _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef PyUnicode_Append(&temp, right_o); *target_local = PyStackRef_FromPyObjectSteal(temp); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (PyStackRef_IsNull(*target_local)) goto pop_2_error; + if (PyStackRef_IsNull(*target_local)) CEVAL_GOTO(pop_2_error); #if TIER_ONE // The STORE_FAST is already done. This is done here in tier one, // and during trace projection in tier two: @@ -704,111 +547,7 @@ _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } @@ -850,128 +589,72 @@ _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *sta ((PyFloatObject *)left_o)->ob_fval * ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) CEVAL_GOTO(pop_2_error); res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP_MULTIPLY_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_OP_MULTIPLY_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_INT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + /* Skip 1 cache entry */ + // _BINARY_OP_MULTIPLY_INT { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_OP_MULTIPLY_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_OP_MULTIPLY_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -979,151 +662,48 @@ _TAIL_CALL_BINARY_OP_MULTIPLY_INT(_PyInterpreterFrame *frame, _PyStackRef *stack { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); _PyStackRef left; _PyStackRef right; _PyStackRef res; - // _GUARD_BOTH_INT + // _GUARD_BOTH_FLOAT { right = stack_pointer[-1]; left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ - // _BINARY_OP_MULTIPLY_INT + // _BINARY_OP_SUBTRACT_FLOAT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) goto pop_2_error; + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_BINARY_OP_SUBTRACT_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_BINARY_OP_SUBTRACT_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -1131,152 +711,110 @@ _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *sta { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); _PyStackRef left; _PyStackRef right; _PyStackRef res; - // _GUARD_BOTH_FLOAT + // _GUARD_BOTH_INT { right = stack_pointer[-1]; left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ - // _BINARY_OP_SUBTRACT_FLOAT + // _BINARY_OP_SUBTRACT_INT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval - - ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) goto pop_2_error; + PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BINARY_SLICE); + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + _PyStackRef res; + // _SPECIALIZE_BINARY_SLICE + { + // Placeholder until we implement BINARY_SLICE specialization + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(BINARY_SLICE); + #endif /* ENABLE_SPECIALIZATION */ } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _BINARY_SLICE { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyObject *res_o; + // Can't use ERROR_IF() here, because we haven't + // DECREF'ed container yet, and we still own slice. + if (slice == NULL) { + res_o = NULL; } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(slice); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } -#endif - DISPATCH(); + PyStackRef_CLOSE(container); + if (res_o == NULL) CEVAL_GOTO(pop_3_error); + res = PyStackRef_FromPyObjectSteal(res_o); } + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_OP_SUBTRACT_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_BINARY_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_OP_SUBTRACT_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_BINARY_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -1284,318 +822,107 @@ _TAIL_CALL_BINARY_OP_SUBTRACT_INT(_PyInterpreterFrame *frame, _PyStackRef *stack { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; + INSTRUCTION_STATS(BINARY_SUBSCR); + PREDICTED(BINARY_SUBSCR); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef container; + _PyStackRef sub; _PyStackRef res; - // _GUARD_BOTH_INT + // _SPECIALIZE_BINARY_SUBSCR { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + assert(frame->stackpointer == NULL); + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_BinarySubscr(container, sub, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(BINARY_SUBSCR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } - /* Skip 1 cache entry */ - // _BINARY_OP_SUBTRACT_INT + // _BINARY_SUBSCR { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) goto pop_2_error; + PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); + PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_GetItem(container_o, sub_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_BINARY_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_BINARY_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BINARY_SLICE); - _PyStackRef container; - _PyStackRef start; - _PyStackRef stop; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef dict_st; + _PyStackRef sub_st; _PyStackRef res; - // _SPECIALIZE_BINARY_SLICE - { - // Placeholder until we implement BINARY_SLICE specialization - #if ENABLE_SPECIALIZATION - OPCODE_DEFERRED_INC(BINARY_SLICE); - #endif /* ENABLE_SPECIALIZATION */ - } - // _BINARY_SLICE - { - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + dict_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int rc = PyDict_GetItemRef(dict, sub, &res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (rc == 0) { _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); + _PyErr_SetKeyError(sub); stack_pointer = _PyFrame_GetStackPointer(frame); - PyObject *res_o; - // Can't use ERROR_IF() here, because we haven't - // DECREF'ed container yet, and we still own slice. - if (slice == NULL) { - res_o = NULL; - } - else { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); - stack_pointer = _PyFrame_GetStackPointer(frame); - Py_DECREF(slice); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - } - PyStackRef_CLOSE(container); - if (res_o == NULL) goto pop_3_error; - res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[-3] = res; - stack_pointer += -2; + PyStackRef_CLOSE(dict_st); + PyStackRef_CLOSE(sub_st); + if (rc <= 0) CEVAL_GOTO(pop_2_error); + // not found or error + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_BINARY_SUBSCR_GETITEM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_BINARY_SUBSCR_GETITEM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -1603,163 +930,169 @@ _TAIL_CALL_BINARY_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR); - PREDICTED(BINARY_SUBSCR); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; + INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); _PyStackRef container; + _PyStackRef getitem; _PyStackRef sub; - _PyStackRef res; - // _SPECIALIZE_BINARY_SUBSCR + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + } + // _BINARY_SUBSCR_CHECK_FUNC { - sub = stack_pointer[-1]; container = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - assert(frame->stackpointer == NULL); - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_BinarySubscr(container, sub, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(BINARY_SUBSCR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); + DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR); + assert(PyFunction_Check(getitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); + DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR); + PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); + assert(code->co_argcount == 2); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + getitem = PyStackRef_FromPyObjectNew(getitem_o); + STAT_INC(BINARY_SUBSCR, hit); } - // _BINARY_SUBSCR + // _BINARY_SUBSCR_INIT_CALL { - PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); - PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); + sub = stack_pointer[-1]; + new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + new_frame->localsplus[0] = container; + new_frame->localsplus[1] = sub; + frame->return_offset = 2 ; + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_GetItem(container_o, sub_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (res_o == NULL) goto pop_2_error; - res = PyStackRef_FromPyObjectSteal(res_o); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } + DISPATCH(); + } +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef list_st; + _PyStackRef sub_st; + _PyStackRef res; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyList_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + stack_pointer = _PyFrame_GetStackPointer(frame); + DEOPT_IF(res_o == NULL, BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + #else + DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + Py_INCREF(res_o); + #endif + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(list_st); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_SUBSCR_STR_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BINARY_SUBSCR_STR_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef str_st; + _PyStackRef sub_st; + _PyStackRef res; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + str_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + // Specialize for reading an ASCII character from any string: + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(str_st); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -1767,1384 +1100,1207 @@ _TAIL_CALL_BINARY_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_poi { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_DICT); + INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef dict_st; + _PyStackRef tuple_st; _PyStackRef sub_st; _PyStackRef res; /* Skip 1 cache entry */ sub_st = stack_pointer[-1]; - dict_st = stack_pointer[-2]; + tuple_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyTuple_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); - PyObject *res_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int rc = PyDict_GetItemRef(dict, sub, &res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (rc == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetKeyError(sub); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - PyStackRef_CLOSE(dict_st); - PyStackRef_CLOSE(sub_st); - if (rc <= 0) goto pop_2_error; - // not found or error + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + Py_INCREF(res_o); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_SUBSCR_GETITEM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_BUILD_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_SUBSCR_GETITEM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_BUILD_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef container; - _PyStackRef getitem; - _PyStackRef sub; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); - } - // _BINARY_SUBSCR_CHECK_FUNC - { - container = stack_pointer[-2]; - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR); - assert(PyFunction_Check(getitem_o)); - uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR); - PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); - assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); - getitem = PyStackRef_FromPyObjectNew(getitem_o); - STAT_INC(BINARY_SUBSCR, hit); - } - // _BINARY_SUBSCR_INIT_CALL - { - sub = stack_pointer[-1]; - new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); - new_frame->localsplus[0] = container; - new_frame->localsplus[1] = sub; - frame->return_offset = 2 ; - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - stack_pointer += -2; + next_instr += 1; + INSTRUCTION_STATS(BUILD_LIST); + _PyStackRef *values; + _PyStackRef list; + values = &stack_pointer[-oparg]; + PyObject *list_o = _PyList_FromStackRefSteal(values, oparg); + if (list_o == NULL) { + stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); + CEVAL_GOTO(error); } + list = PyStackRef_FromPyObjectSteal(list_o); + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BUILD_MAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BUILD_MAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_MAP); + _PyStackRef *values; + _PyStackRef map; + values = &stack_pointer[-oparg*2]; + STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); + if (CONVERSION_FAILED(values_o)) { + for (int _i = oparg*2; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + { + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *map_o = _PyDict_FromItems( + values_o, 2, + values_o+1, 2, + oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + for (int _i = oparg*2; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + if (map_o == NULL) { + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); + } + map = PyStackRef_FromPyObjectSteal(map_o); + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BUILD_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BUILD_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SET); + _PyStackRef *values; + _PyStackRef set; + values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *set_o = PySet_New(NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (set_o == NULL) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); + { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; + } + int err = 0; + for (int i = 0; i < oparg; i++) { + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PySet_Add(set_o, PyStackRef_AsPyObjectBorrow(values[i])); + stack_pointer = _PyFrame_GetStackPointer(frame); } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + PyStackRef_CLOSE(values[i]); + } + if (err != 0) { + Py_DECREF(set_o); + { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } -#endif - DISPATCH(); } + set = PyStackRef_FromPyObjectSteal(set_o); + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_BUILD_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_BUILD_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef list_st; - _PyStackRef sub_st; - _PyStackRef res; - /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - #ifdef Py_GIL_DISABLED - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); - stack_pointer = _PyFrame_GetStackPointer(frame); - DEOPT_IF(res_o == NULL, BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - #else - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - PyObject *res_o = PyList_GET_ITEM(list, index); - assert(res_o != NULL); - Py_INCREF(res_o); - #endif - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - PyStackRef_CLOSE(list_st); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SLICE); + _PyStackRef start; + _PyStackRef stop; + _PyStackRef step = PyStackRef_NULL; + _PyStackRef slice; + if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } + stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; + start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; + PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); + PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); + PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); + PyStackRef_CLOSE(start); + PyStackRef_CLOSE(stop); + PyStackRef_XCLOSE(step); + if (slice_o == NULL) { + stack_pointer += -2 - ((oparg == 3) ? 1 : 0); + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); + } + slice = PyStackRef_FromPyObjectSteal(slice_o); + stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; + stack_pointer += -1 - ((oparg == 3) ? 1 : 0); assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BUILD_STRING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BUILD_STRING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_STRING); + _PyStackRef *pieces; + _PyStackRef str; + pieces = &stack_pointer[-oparg]; + STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); + if (CONVERSION_FAILED(pieces_o)) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(pieces[_i]); } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); + { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } + } + PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(pieces[_i]); + } + if (str_o == NULL) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); + } + str = PyStackRef_FromPyObjectSteal(str_o); + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BUILD_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_BUILD_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TUPLE); + _PyStackRef *values; + _PyStackRef tup; + values = &stack_pointer[-oparg]; + PyObject *tup_o = _PyTuple_FromStackRefSteal(values, oparg); + if (tup_o == NULL) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } + tup = PyStackRef_FromPyObjectSteal(tup_o); + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_SUBSCR_STR_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CACHE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_SUBSCR_STR_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CACHE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef str_st; - _PyStackRef sub_st; - _PyStackRef res; - /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - str_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); - // Specialize for reading an ASCII character from any string: - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - PyStackRef_CLOSE(str_st); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + next_instr += 1; + INSTRUCTION_STATS(CACHE); + assert(0 && "Executing a cache."); + Py_FatalError("Executing a cache."); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL); + PREDICTED(CALL); + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef res; + // _SPECIALIZE_CALL + { + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; + _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); + OPCODE_DEFERRED_INC(CALL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD + { + args = &stack_pointer[-oparg]; + func = &stack_pointer[-2 - oparg]; + maybe_self = &stack_pointer[-1 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; + } + // _DO_CALL + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + CEVAL_GOTO(error); } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); + frame->return_offset = 4 ; + DISPATCH_INLINED(new_frame); } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); + } } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(res_o); + } + } } -#endif - DISPATCH(); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef tuple_st; - _PyStackRef sub_st; - _PyStackRef res; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef *args; + _PyStackRef *init; + _PyStackRef *self; + _PyInterpreterFrame *init_frame; + _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - tuple_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - PyObject *res_o = PyTuple_GET_ITEM(tuple, index); - assert(res_o != NULL); - Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - PyStackRef_CLOSE(tuple_st); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); + // _CHECK_AND_ALLOCATE_OBJECT + { + args = &stack_pointer[-oparg]; + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + init = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL); + PyTypeObject *tp = (PyTypeObject *)callable_o; + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL); + assert(tp->tp_new == PyBaseObject_Type.tp_new); + assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_alloc == PyType_GenericAlloc); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; + PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); + PyCodeObject *code = (PyCodeObject *)init_func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *self_o = PyType_GenericAlloc(tp, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (self_o == NULL) { + CEVAL_GOTO(error); } + self[0] = PyStackRef_FromPyObjectSteal(self_o); + _PyStackRef temp = callable[0]; + init[0] = PyStackRef_FromPyObjectNew(init_func); + PyStackRef_CLOSE(temp); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _CREATE_INIT_FRAME { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + args = &stack_pointer[-oparg]; + self = &stack_pointer[-1 - oparg]; + init = &stack_pointer[-2 - oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); + stack_pointer = _PyFrame_GetStackPointer(frame); + /* Push self onto stack of shim */ + shim->localsplus[0] = PyStackRef_DUP(self[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, init[0], NULL, args-1, oparg+1, NULL, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + _PyEval_FrameClearAndPop(tstate, shim); + CEVAL_GOTO(error); } -#endif - DISPATCH(); + init_frame = temp; + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; + /* Account for pushing the extra frame. + * We don't check recursion depth here, + * as it will be checked after start_frame */ + tstate->py_recursion_remaining--; } + // _PUSH_FRAME + { + new_frame = init_frame; + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BUILD_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BUILD_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_LIST); - _PyStackRef *values; - _PyStackRef list; - values = &stack_pointer[-oparg]; - PyObject *list_o = _PyList_FromStackRefSteal(values, oparg); - if (list_o == NULL) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef *func; + _PyStackRef *self; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); } - list = PyStackRef_FromPyObjectSteal(list_o); - stack_pointer[-oparg] = list; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + { + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL); } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + func = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + STAT_INC(CALL, hit); + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + PyStackRef_CLOSE(temp); + } + // flush + // _CHECK_FUNCTION_VERSION + { + callable = &stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL); + } + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null[0]); + STAT_INC(CALL, hit); + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null[0]; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _SAVE_RETURN_OFFSET { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BUILD_MAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BUILD_MAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_MAP); - _PyStackRef *values; - _PyStackRef map; - values = &stack_pointer[-oparg*2]; - STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); - if (CONVERSION_FAILED(values_o)) { - for (int _i = oparg*2; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef *method; + _PyStackRef *self; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_METHOD_VERSION + { + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL); + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + DEOPT_IF(!PyFunction_Check(func), CALL); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + } + // _EXPAND_METHOD + { + method = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + assert(PyStackRef_IsNull(null[0])); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable[0]; + method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(method[0])); + PyStackRef_CLOSE(temp); + } + // flush + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; } - { - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + // The frame has stolen all the arguments from the stack. + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + CEVAL_GOTO(error); } + new_frame = temp; } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *map_o = _PyDict_FromItems( - values_o, 2, - values_o+1, 2, - oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); - for (int _i = oparg*2; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } - if (map_o == NULL) { - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - goto error; + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } - map = PyStackRef_FromPyObjectSteal(map_o); - stack_pointer[-oparg*2] = map; - stack_pointer += 1 - oparg*2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_BUILTIN_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_BUILTIN_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_CLASS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_CLASS { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; + DEOPT_IF(!PyType_Check(callable_o), CALL); + PyTypeObject *tp = (PyTypeObject *)callable_o; + DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + STAT_INC(CALL, hit); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } -#endif - DISPATCH(); + res = PyStackRef_FromPyObjectSteal(res_o); } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BUILD_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_BUILTIN_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BUILD_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_BUILTIN_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SET); - _PyStackRef *values; - _PyStackRef set; - values = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *set_o = PySet_New(NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (set_o == NULL) { - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* Builtin METH_FASTCALL functions, without keywords */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; } - { - stack_pointer += -oparg; + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + /* res = func(self, args, nargs) */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable_o), + args_o, + total_args); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } + res = PyStackRef_FromPyObjectSteal(res_o); } - int err = 0; - for (int i = 0; i < oparg; i++) { - if (err == 0) { + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - err = PySet_Add(set_o, PyStackRef_AsPyObjectBorrow(values[i])); + int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - } - PyStackRef_CLOSE(values[i]); - } - if (err != 0) { - Py_DECREF(set_o); - { - stack_pointer += -oparg; + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; } } - set = PyStackRef_FromPyObjectSteal(set_o); - stack_pointer[-oparg] = set; - stack_pointer += 1 - oparg; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST_WITH_KEYWORDS { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); + STAT_INC(CALL, hit); + /* res = func(self, args, nargs, kwnames) */ + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } -#endif - DISPATCH(); + res = PyStackRef_FromPyObjectSteal(res_o); } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BUILD_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_BUILTIN_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BUILD_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_BUILTIN_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SLICE); - _PyStackRef start; - _PyStackRef stop; - _PyStackRef step = PyStackRef_NULL; - _PyStackRef slice; - if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } - stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; - start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; - PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); - PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); - PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); - PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); - PyStackRef_CLOSE(start); - PyStackRef_CLOSE(stop); - PyStackRef_XCLOSE(step); - if (slice_o == NULL) { - stack_pointer += -2 - ((oparg == 3) ? 1 : 0); - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - slice = PyStackRef_FromPyObjectSteal(slice_o); - stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; - stack_pointer += -1 - ((oparg == 3) ? 1 : 0); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_O + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* Builtin METH_O functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + _PyStackRef arg = args[0]; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(arg); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } + res = PyStackRef_FromPyObjectSteal(res_o); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _CHECK_PERIODIC { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BUILD_STRING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_BUILD_STRING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -3152,19624 +2308,1727 @@ _TAIL_CALL_BUILD_STRING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(BUILD_STRING); - _PyStackRef *pieces; - _PyStackRef str; - pieces = &stack_pointer[-oparg]; - STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); - if (CONVERSION_FAILED(pieces_o)) { - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(pieces[_i]); - } - { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; + INSTRUCTION_STATS(CALL_FUNCTION_EX); + PREDICTED(CALL_FUNCTION_EX); + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef kwargs_in = PyStackRef_NULL; + _PyStackRef tuple; + _PyStackRef kwargs_out = PyStackRef_NULL; + _PyStackRef func_st; + _PyStackRef callargs_st; + _PyStackRef kwargs_st = PyStackRef_NULL; + _PyStackRef result; + // _MAKE_CALLARGS_A_TUPLE + { + if (oparg & 1) { kwargs_in = stack_pointer[-(oparg & 1)]; } + callargs = stack_pointer[-1 - (oparg & 1)]; + func = stack_pointer[-3 - (oparg & 1)]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (PyTuple_CheckExact(callargs_o)) { + tuple = callargs; } - } - PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(pieces[_i]); - } - if (str_o == NULL) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - str = PyStackRef_FromPyObjectSteal(str_o); - stack_pointer[-oparg] = str; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + CEVAL_GOTO(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { + CEVAL_GOTO(error); + } + PyStackRef_CLOSE(callargs); + tuple = PyStackRef_FromPyObjectSteal(tuple_o); } + kwargs_out = kwargs_in; } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _DO_CALL_FUNCTION_EX { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); + kwargs_st = kwargs_out; + callargs_st = tuple; + func_st = func; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-1 - (oparg & 1)] = callargs_st; + if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + CEVAL_GOTO(error); } - assert(STACK_LEVEL() == 0); _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(result_o); + } + } } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_BUILD_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_BUILD_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_TUPLE); - _PyStackRef *values; - _PyStackRef tup; - values = &stack_pointer[-oparg]; - PyObject *tup_o = _PyTuple_FromStackRefSteal(values, oparg); - if (tup_o == NULL) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - tup = PyStackRef_FromPyObjectSteal(tup_o); - stack_pointer[-oparg] = tup; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CACHE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CACHE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CACHE); - assert(0 && "Executing a cache."); - Py_FatalError("Executing a cache."); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL); - PREDICTED(CALL); - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef *func; - _PyStackRef *maybe_self; - _PyStackRef res; - // _SPECIALIZE_CALL - { - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CALL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD - { - args = &stack_pointer[-oparg]; - func = &stack_pointer[-2 - oparg]; - maybe_self = &stack_pointer[-1 - oparg]; - if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); - PyStackRef_CLOSE(temp); - } - } - // _DO_CALL - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - // Check if the call can be inlined or not - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - args, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - goto error; - } - frame->return_offset = 4 ; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(res_o); - } - } - } - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *null; - _PyStackRef *args; - _PyStackRef *init; - _PyStackRef *self; - _PyInterpreterFrame *init_frame; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_AND_ALLOCATE_OBJECT - { - args = &stack_pointer[-oparg]; - null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - init = &stack_pointer[-2 - oparg]; - self = &stack_pointer[-1 - oparg]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(!PyType_Check(callable_o), CALL); - PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL); - assert(tp->tp_new == PyBaseObject_Type.tp_new); - assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); - assert(tp->tp_alloc == PyType_GenericAlloc); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; - PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); - PyCodeObject *code = (PyCodeObject *)init_func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *self_o = PyType_GenericAlloc(tp, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (self_o == NULL) { - goto error; - } - self[0] = PyStackRef_FromPyObjectSteal(self_o); - _PyStackRef temp = callable[0]; - init[0] = PyStackRef_FromPyObjectNew(init_func); - PyStackRef_CLOSE(temp); - } - // _CREATE_INIT_FRAME - { - args = &stack_pointer[-oparg]; - self = &stack_pointer[-1 - oparg]; - init = &stack_pointer[-2 - oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); - assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); - assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); - stack_pointer = _PyFrame_GetStackPointer(frame); - /* Push self onto stack of shim */ - shim->localsplus[0] = PyStackRef_DUP(self[0]); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, init[0], NULL, args-1, oparg+1, NULL, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - _PyEval_FrameClearAndPop(tstate, shim); - goto error; - } - init_frame = temp; - frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; - /* Account for pushing the extra frame. - * We don't check recursion depth here, - * as it will be checked after start_frame */ - tstate->py_recursion_remaining--; - } - // _PUSH_FRAME - { - new_frame = init_frame; - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *null; - _PyStackRef *func; - _PyStackRef *self; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS - { - null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL); - } - // _INIT_CALL_BOUND_METHOD_EXACT_ARGS - { - func = &stack_pointer[-2 - oparg]; - self = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - STAT_INC(CALL, hit); - self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - PyStackRef_CLOSE(temp); - } - // flush - // _CHECK_FUNCTION_VERSION - { - callable = &stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); - } - // _CHECK_FUNCTION_EXACT_ARGS - { - self_or_null = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - assert(PyFunction_Check(callable_o)); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); - } - // _CHECK_STACK_SPACE - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); - } - // _INIT_CALL_PY_EXACT_ARGS - { - args = &stack_pointer[-oparg]; - int has_self = !PyStackRef_IsNull(self_or_null[0]); - STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null[0]; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; - } - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *null; - _PyStackRef *method; - _PyStackRef *self; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_METHOD_VERSION - { - null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL); - PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - } - // _EXPAND_METHOD - { - method = &stack_pointer[-2 - oparg]; - self = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - assert(PyStackRef_IsNull(null[0])); - assert(Py_TYPE(callable_o) == &PyMethod_Type); - self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - _PyStackRef temp = callable[0]; - method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(method[0])); - PyStackRef_CLOSE(temp); - } - // flush - // _PY_FRAME_GENERAL - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - args, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - // The frame has stolen all the arguments from the stack. - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - goto error; - } - new_frame = temp; - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_BUILTIN_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_BUILTIN_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_CLASS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_CLASS - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - DEOPT_IF(!PyType_Check(callable_o), CALL); - PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); - STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_BUILTIN_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_BUILTIN_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL functions, without keywords */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - /* res = func(self, args, nargs) */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable_o), - args_o, - total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST_WITH_KEYWORDS - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); - STAT_INC(CALL, hit); - /* res = func(self, args, nargs, kwnames) */ - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_BUILTIN_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_BUILTIN_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_O); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_O - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - /* Builtin METH_O functions */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL); - // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - _PyStackRef arg = args[0]; - _Py_EnterRecursiveCallTstateUnchecked(tstate); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(arg); - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_FUNCTION_EX); - PREDICTED(CALL_FUNCTION_EX); - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; - _PyStackRef func; - _PyStackRef callargs; - _PyStackRef kwargs_in = PyStackRef_NULL; - _PyStackRef tuple; - _PyStackRef kwargs_out = PyStackRef_NULL; - _PyStackRef func_st; - _PyStackRef callargs_st; - _PyStackRef kwargs_st = PyStackRef_NULL; - _PyStackRef result; - // _MAKE_CALLARGS_A_TUPLE - { - if (oparg & 1) { kwargs_in = stack_pointer[-(oparg & 1)]; } - callargs = stack_pointer[-1 - (oparg & 1)]; - func = stack_pointer[-3 - (oparg & 1)]; - PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); - if (PyTuple_CheckExact(callargs_o)) { - tuple = callargs; - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (tuple_o == NULL) { - goto error; - } - PyStackRef_CLOSE(callargs); - tuple = PyStackRef_FromPyObjectSteal(tuple_o); - } - kwargs_out = kwargs_in; - } - // _DO_CALL_FUNCTION_EX - { - kwargs_st = kwargs_out; - callargs_st = tuple; - func_st = func; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - PyObject *result_o; - assert(!_PyErr_Occurred(tstate)); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - assert(PyTuple_CheckExact(callargs)); - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - stack_pointer[-1 - (oparg & 1)] = callargs_st; - if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(result_o); - } - } - } - } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -2 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( - tstate, func_st, locals, - nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - // Need to sync the stack since we exit with DISPATCH_INLINED. - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { - goto error; - } - assert( 1 == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); - } - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[-1 - (oparg & 1)] = callargs_st; - if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(callargs_st); - PyStackRef_CLOSE(func_st); - if (result_o == NULL) { - stack_pointer += -3 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - result = PyStackRef_FromPyObjectSteal(result_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-3 - (oparg & 1)] = result; - stack_pointer += -2 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 2 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-3 - (oparg & 1)] = result; - stack_pointer += -2 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_INTRINSIC_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_INTRINSIC_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_1); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - assert(oparg <= MAX_INTRINSIC_1); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (res_o == NULL) goto pop_1_error; - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_INTRINSIC_2(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_INTRINSIC_2(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_2); - _PyStackRef value2_st; - _PyStackRef value1_st; - _PyStackRef res; - value1_st = stack_pointer[-1]; - value2_st = stack_pointer[-2]; - assert(oparg <= MAX_INTRINSIC_2); - PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); - PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value2_st); - PyStackRef_CLOSE(value1_st); - if (res_o == NULL) goto pop_2_error; - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_ISINSTANCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_ISINSTANCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ISINSTANCE); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - /* isinstance(o, o2) */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - DEOPT_IF(total_args != 2, CALL); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL); - STAT_INC(CALL, hit); - _PyStackRef cls_stackref = args[1]; - _PyStackRef inst_stackref = args[0]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (retval < 0) { - goto error; - } - res = retval ? PyStackRef_True : PyStackRef_False; - assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(inst_stackref); - PyStackRef_CLOSE(cls_stackref); - PyStackRef_CLOSE(callable[0]); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW); - PREDICTED(CALL_KW); - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef kwnames_in; - _PyStackRef *func; - _PyStackRef *maybe_self; - _PyStackRef kwnames_out; - _PyStackRef res; - // _SPECIALIZE_CALL_KW - { - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CALL_KW); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD_KW - { - kwnames_in = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - func = &stack_pointer[-3 - oparg]; - maybe_self = &stack_pointer[-2 - oparg]; - if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); - PyStackRef_CLOSE(temp); - } - kwnames_out = kwnames_in; - } - // _DO_CALL_KW - { - kwnames = kwnames_out; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - // Check if the call can be inlined or not - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-1] = kwnames; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - args, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(kwnames); - // Sync stack explicitly since we leave using DISPATCH_INLINED(). - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - goto error; - } - assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); - frame->return_offset = 4 ; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - PyStackRef_CLOSE(kwnames); - { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - } - stack_pointer[-1] = kwnames; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(res_o); - } - } - } - PyStackRef_CLOSE(kwnames); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - if (res_o == NULL) { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_KW_BOUND_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_KW_BOUND_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *null; - _PyStackRef kwnames; - _PyStackRef *method; - _PyStackRef *self; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW); - } - // _CHECK_METHOD_VERSION_KW - { - null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW); - PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL_KW); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW); - } - // _EXPAND_METHOD_KW - { - method = &stack_pointer[-3 - oparg]; - self = &stack_pointer[-2 - oparg]; - _PyStackRef callable_s = callable[0]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable_s); - assert(PyStackRef_IsNull(null[0])); - assert(Py_TYPE(callable_o) == &PyMethod_Type); - self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(method[0])); - PyStackRef_CLOSE(callable_s); - } - // flush - // _PY_FRAME_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - args, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(kwnames); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - goto error; - } - new_frame = temp; - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_KW_NON_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_KW_NON_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_NON_PY); - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef kwnames; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CHECK_IS_NOT_PY_CALLABLE_KW - { - callable = &stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL_KW); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW); - } - // _CALL_KW_NON_PY - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - #if TIER_ONE - assert(opcode != INSTRUMENTED_CALL); - #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - PyStackRef_CLOSE(kwnames); - { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(kwnames); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 2 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_KW_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_KW_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_PY); - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef kwnames; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW); - } - // _CHECK_FUNCTION_VERSION_KW - { - callable = &stack_pointer[-3 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL_KW); - } - // _PY_FRAME_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - args, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(kwnames); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - goto error; - } - new_frame = temp; - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LEN); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - /* len(o) */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.len, CALL); - STAT_INC(CALL, hit); - _PyStackRef arg_stackref = args[0]; - PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_ssize_t len_i = PyObject_Length(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) { - goto error; - } - PyObject *res_o = PyLong_FromSsize_t(len_i); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res_o == NULL) { - GOTO_ERROR(error); - } - PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(arg_stackref); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LIST_APPEND); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self; - _PyStackRef arg; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - arg = stack_pointer[-1]; - self = stack_pointer[-2]; - callable = stack_pointer[-3]; - assert(oparg == 1); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL); - assert(self_o != NULL); - DEOPT_IF(!PyList_Check(self_o), CALL); - DEOPT_IF(!LOCK_OBJECT(self_o), CALL); - STAT_INC(CALL, hit); - int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); - UNLOCK_OBJECT(self_o); - PyStackRef_CLOSE(self); - PyStackRef_CLOSE(callable); - if (err) goto pop_3_error; - #if TIER_ONE - // Skip the following POP_TOP. This is done here in tier one, and - // during trace projection in tier two: - assert(next_instr->op.code == POP_TOP); - SKIP_OVER(1); - #endif - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); - PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFast cfunc = - (PyCFunctionFast)(void(*)(void))meth->ml_meth; - PyObject *res_o = cfunc(self, (args_o + 1), nargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); - PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); - STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_NOARGS - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - _PyStackRef self_stackref = args[0]; - PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); - // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - _Py_EnterRecursiveCallTstateUnchecked(tstate); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(self_stackref); - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_O - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); - // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); - _PyStackRef arg_stackref = args[1]; - _PyStackRef self_stackref = args[0]; - DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - _Py_EnterRecursiveCallTstateUnchecked(tstate); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, - PyStackRef_AsPyObjectBorrow(self_stackref), - PyStackRef_AsPyObjectBorrow(arg_stackref)); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(self_stackref); - PyStackRef_CLOSE(arg_stackref); - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_NON_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_NON_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_NON_PY_GENERAL); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CHECK_IS_NOT_PY_CALLABLE - { - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL); - } - // _CALL_NON_PY_GENERAL - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - #if TIER_ONE - assert(opcode != INSTRUMENTED_CALL); - #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_PY_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_PY_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_FUNCTION_VERSION - { - callable = &stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); - } - // _CHECK_FUNCTION_EXACT_ARGS - { - self_or_null = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - assert(PyFunction_Check(callable_o)); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); - } - // _CHECK_STACK_SPACE - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); - } - // _INIT_CALL_PY_EXACT_ARGS - { - args = &stack_pointer[-oparg]; - int has_self = !PyStackRef_IsNull(self_or_null[0]); - STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null[0]; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; - } - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_GENERAL); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_FUNCTION_VERSION - { - callable = &stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); - } - // _PY_FRAME_GENERAL - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - args, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - // The frame has stolen all the arguments from the stack. - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - goto error; - } - new_frame = temp; - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_STR_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_STR_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_STR_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_STR_1 - { - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL); - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Str(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(arg); - if (res_o == NULL) goto pop_3_error; - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-3] = res; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-3] = res; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_TUPLE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_TUPLE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TUPLE_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_TUPLE_1 - { - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL); - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PySequence_Tuple(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(arg); - if (res_o == NULL) goto pop_3_error; - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-3] = res; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-3] = res; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_TYPE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CALL_TYPE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TYPE_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL); - STAT_INC(CALL, hit); - res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); - PyStackRef_CLOSE(arg); - stack_pointer[-3] = res; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CHECK_EG_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CHECK_EG_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EG_MATCH); - _PyStackRef exc_value_st; - _PyStackRef match_type_st; - _PyStackRef rest; - _PyStackRef match; - match_type_st = stack_pointer[-1]; - exc_value_st = stack_pointer[-2]; - PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - PyStackRef_CLOSE(exc_value_st); - PyStackRef_CLOSE(match_type_st); - goto pop_2_error; - } - PyObject *match_o = NULL; - PyObject *rest_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match_o, &rest_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(exc_value_st); - PyStackRef_CLOSE(match_type_st); - if (res < 0) goto pop_2_error; - assert((match_o == NULL) == (rest_o == NULL)); - if (match_o == NULL) goto pop_2_error; - if (!Py_IsNone(match_o)) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyErr_SetHandledException(match_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - } - rest = PyStackRef_FromPyObjectSteal(rest_o); - match = PyStackRef_FromPyObjectSteal(match_o); - stack_pointer[-2] = rest; - stack_pointer[-1] = match; - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CHECK_EXC_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CHECK_EXC_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EXC_MATCH); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyExceptionInstance_Check(left_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyEval_CheckExceptTypeValid(tstate, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - PyStackRef_CLOSE(right); - goto pop_1_error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PyErr_GivenExceptionMatches(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(right); - b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[-1] = b; - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CLEANUP_THROW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CLEANUP_THROW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(CLEANUP_THROW); - _PyStackRef sub_iter_st; - _PyStackRef last_sent_val_st; - _PyStackRef exc_value_st; - _PyStackRef none; - _PyStackRef value; - exc_value_st = stack_pointer[-1]; - last_sent_val_st = stack_pointer[-2]; - sub_iter_st = stack_pointer[-3]; - PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - // assert(throwflag); - assert(exc_value && PyExceptionInstance_Check(exc_value)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - none = PyStackRef_None; - value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); - PyStackRef_CLOSE(sub_iter_st); - PyStackRef_CLOSE(last_sent_val_st); - PyStackRef_CLOSE(exc_value_st); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); - monitor_reraise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto exception_unwind; - } - stack_pointer[-3] = none; - stack_pointer[-2] = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_COMPARE_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_COMPARE_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP); - PREDICTED(COMPARE_OP); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _SPECIALIZE_COMPARE_OP - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(COMPARE_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ - } - // _COMPARE_OP - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert((oparg >> 5) <= Py_GE); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - if (res_o == NULL) goto pop_2_error; - if (oparg & 16) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res_bool = PyObject_IsTrue(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - Py_DECREF(res_o); - if (res_bool < 0) goto error; - res = res_bool ? PyStackRef_True : PyStackRef_False; - } - else { - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_COMPARE_OP_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_COMPARE_OP_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_FLOAT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP); - } - /* Skip 1 cache entry */ - // _COMPARE_OP_FLOAT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left_o); - double dright = PyFloat_AS_DOUBLE(right_o); - // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg - int sign_ish = COMPARISON_BIT(dleft, dright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - // It's always a bool, so we don't care about oparg & 16. - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_COMPARE_OP_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_COMPARE_OP_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_INT); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_INT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP); - } - /* Skip 1 cache entry */ - // _COMPARE_OP_INT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && - _PyLong_DigitCount((PyLongObject *)right_o) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); - // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg - int sign_ish = COMPARISON_BIT(ileft, iright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - // It's always a bool, so we don't care about oparg & 16. - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_COMPARE_OP_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_COMPARE_OP_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_STR); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_UNICODE - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP); - } - /* Skip 1 cache entry */ - // _COMPARE_OP_STR - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left_o, right_o); - assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(eq == 0 || eq == 1); - assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); - assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; - // It's always a bool, so we don't care about oparg & 16. - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CONTAINS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CONTAINS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP); - PREDICTED(CONTAINS_OP); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - // _SPECIALIZE_CONTAINS_OP - { - right = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ContainsOp(right, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CONTAINS_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _CONTAINS_OP - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PySequence_Contains(right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - if (res < 0) goto pop_2_error; - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[-2] = b; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CONTAINS_OP_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CONTAINS_OP_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP_DICT); - static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - /* Skip 1 cache entry */ - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP); - STAT_INC(CONTAINS_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PyDict_Contains(right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - if (res < 0) goto pop_2_error; - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CONTAINS_OP_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CONTAINS_OP_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP_SET); - static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - /* Skip 1 cache entry */ - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP); - STAT_INC(CONTAINS_OP, hit); - // Note: both set and frozenset use the same seq_contains method! - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PySet_Contains((PySetObject *)right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - if (res < 0) goto pop_2_error; - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CONVERT_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_CONVERT_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CONVERT_VALUE); - _PyStackRef value; - _PyStackRef result; - value = stack_pointer[-1]; - conversion_func conv_fn; - assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = _PyEval_ConversionFuncs[oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (result_o == NULL) goto pop_1_error; - result = PyStackRef_FromPyObjectSteal(result_o); - stack_pointer[-1] = result; - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_COPY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_COPY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY); - _PyStackRef bottom; - _PyStackRef top; - bottom = stack_pointer[-1 - (oparg-1)]; - assert(oparg > 0); - top = PyStackRef_DUP(bottom); - stack_pointer[0] = top; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_COPY_FREE_VARS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_COPY_FREE_VARS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY_FREE_VARS); - /* Copy closure variables to free variables */ - PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - PyObject *closure = func->func_closure; - assert(oparg == co->co_nfreevars); - int offset = co->co_nlocalsplus - oparg; - for (int i = 0; i < oparg; ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); - } - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_DELETE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_DELETE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_ATTR); - _PyStackRef owner; - owner = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(owner); - if (err) goto pop_1_error; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_DELETE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_DELETE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_DEREF); - PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - // Can't use ERROR_IF here. - // Fortunately we don't need its superpower. - PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); - if (oldobj == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - Py_DECREF(oldobj); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_DELETE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_DELETE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_FAST); - _PyStackRef v = GETLOCAL(oparg); - if (PyStackRef_IsNull(v)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - SETLOCAL(oparg, PyStackRef_NULL); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_DELETE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_DELETE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_GLOBAL); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Pop(GLOBALS(), name, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - // Can't use ERROR_IF here. - if (err < 0) { - goto error; - } - if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_DELETE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_DELETE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_NAME); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals when deleting %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_DelItem(ns, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - // Can't use ERROR_IF here. - if (err != 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_DELETE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_DELETE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_SUBSCR); - _PyStackRef container; - _PyStackRef sub; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - /* del container[sub] */ - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), - PyStackRef_AsPyObjectBorrow(sub)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (err) goto pop_2_error; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_DICT_MERGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_DICT_MERGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_MERGE); - _PyStackRef callable; - _PyStackRef dict; - _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - callable = stack_pointer[-5 - (oparg - 1)]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_MergeEx(dict_o, update_o, 2); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatKwargsError(tstate, callable_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(update); - goto pop_1_error; - } - PyStackRef_CLOSE(update); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_DICT_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_DICT_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_UPDATE); - _PyStackRef dict; - _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Update(dict_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update_o)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - PyStackRef_CLOSE(update); - goto pop_1_error; - } - PyStackRef_CLOSE(update); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_END_ASYNC_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_END_ASYNC_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(END_ASYNC_FOR); - _PyStackRef awaitable_st; - _PyStackRef exc_st; - exc_st = stack_pointer[-1]; - awaitable_st = stack_pointer[-2]; - PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); - assert(exc && PyExceptionInstance_Check(exc)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - PyStackRef_CLOSE(awaitable_st); - PyStackRef_CLOSE(exc_st); - } - else { - Py_INCREF(exc); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto exception_unwind; - } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_FOR); - _PyStackRef value; - value = stack_pointer[-1]; - PyStackRef_CLOSE(value); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_SEND); - _PyStackRef receiver; - _PyStackRef value; - _PyStackRef val; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - (void)receiver; - val = value; - PyStackRef_CLOSE(receiver); - stack_pointer[-2] = val; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_ENTER_EXECUTOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_ENTER_EXECUTOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(ENTER_EXECUTOR); - #ifdef _Py_TIER2 - PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; - assert(executor->vm_data.index == INSTR_OFFSET() - 1); - assert(executor->vm_data.code == code); - assert(executor->vm_data.valid); - assert(tstate->previous_executor == NULL); - /* If the eval breaker is set then stay in tier 1. - * This avoids any potentially infinite loops - * involving _RESUME_CHECK */ - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - opcode = executor->vm_data.opcode; - oparg = (oparg & ~255) | executor->vm_data.oparg; - next_instr = this_instr; - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - } - DISPATCH_GOTO(); - } - tstate->previous_executor = Py_None; - Py_INCREF(executor); - GOTO_TIER_TWO(executor); - #else - Py_FatalError("ENTER_EXECUTOR is not supported in this build"); - #endif /* _Py_TIER2 */ - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_EXIT_INIT_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_EXIT_INIT_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXIT_INIT_CHECK); - _PyStackRef should_be_none; - should_be_none = stack_pointer[-1]; - assert(STACK_LEVEL() == 2); - if (!PyStackRef_IsNone(should_be_none)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyErr_Format(PyExc_TypeError, - "__init__() should return None, not '%.200s'", - Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_EXTENDED_ARG(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_EXTENDED_ARG(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXTENDED_ARG); - assert(oparg); - opcode = next_instr->op.code; - oparg = oparg << 8 | next_instr->op.arg; - PRE_DISPATCH_GOTO(); - DISPATCH_GOTO(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_FORMAT_SIMPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_FORMAT_SIMPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_SIMPLE); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - /* If value is a unicode object, then we know the result - * of format(value) is value itself. */ - if (!PyUnicode_CheckExact(value_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Format(value_o, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (res_o == NULL) goto pop_1_error; - res = PyStackRef_FromPyObjectSteal(res_o); - } - else { - res = value; - } - stack_pointer[-1] = res; - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_FORMAT_WITH_SPEC(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_FORMAT_WITH_SPEC(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_WITH_SPEC); - _PyStackRef value; - _PyStackRef fmt_spec; - _PyStackRef res; - fmt_spec = stack_pointer[-1]; - value = stack_pointer[-2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - PyStackRef_CLOSE(fmt_spec); - if (res_o == NULL) goto pop_2_error; - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER); - PREDICTED(FOR_ITER); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef iter; - _PyStackRef next; - // _SPECIALIZE_FOR_ITER - { - iter = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ForIter(iter, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(FOR_ITER); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ - } - // _FOR_ITER - { - /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_o == NULL) { - if (_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - PyStackRef_CLOSE(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */ - JUMPBY(oparg + 2); - DISPATCH(); - } - next = PyStackRef_FromPyObjectSteal(next_o); - // Common case: no jump, leave it to the code generator - } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_FOR_ITER_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_FOR_ITER_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_GEN); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyInterpreterFrame *gen_frame; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); - } - // _FOR_ITER_GEN_FRAME - { - iter = stack_pointer[-1]; - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); - STAT_INC(FOR_ITER, hit); - gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, PyStackRef_None); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - gen_frame->previous = frame; - // oparg is the return offset from the next instruction. - frame->return_offset = (uint16_t)( 2 + oparg); - } - // _PUSH_FRAME - { - new_frame = gen_frame; - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_FOR_ITER_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_FOR_ITER_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_LIST); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_LIST - { - iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER); - } - // _ITER_JUMP_LIST - { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - STAT_INC(FOR_ITER, hit); - PyListObject *seq = it->it_seq; - if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { - it->it_index = -1; - #ifndef Py_GIL_DISABLED - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); - } - #endif - PyStackRef_CLOSE(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ - JUMPBY(oparg + 2); - DISPATCH(); - } - } - // _ITER_NEXT_LIST - { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyList_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); - } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_FOR_ITER_RANGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_FOR_ITER_RANGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_RANGE); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_RANGE - { - iter = stack_pointer[-1]; - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); - } - // _ITER_JUMP_RANGE - { - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - STAT_INC(FOR_ITER, hit); - if (r->len <= 0) { - STACK_SHRINK(1); - PyStackRef_CLOSE(iter); - // Jump over END_FOR and POP_TOP instructions. - JUMPBY(oparg + 2); - DISPATCH(); - } - } - // _ITER_NEXT_RANGE - { - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - assert(r->len > 0); - long value = r->start; - r->start = value + r->step; - r->len--; - PyObject *res = PyLong_FromLong(value); - if (res == NULL) goto error; - next = PyStackRef_FromPyObjectSteal(res); - } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_FOR_ITER_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_FOR_ITER_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_TUPLE); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_TUPLE - { - iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER); - } - // _ITER_JUMP_TUPLE - { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - STAT_INC(FOR_ITER, hit); - PyTupleObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); - } - PyStackRef_CLOSE(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ - JUMPBY(oparg + 2); - DISPATCH(); - } - } - // _ITER_NEXT_TUPLE - { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - PyTupleObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); - } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_GET_AITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_GET_AITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AITER); - _PyStackRef obj; - _PyStackRef iter; - obj = stack_pointer[-1]; - unaryfunc getter = NULL; - PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); - PyObject *iter_o; - PyTypeObject *type = Py_TYPE(obj_o); - if (type->tp_as_async != NULL) { - getter = type->tp_as_async->am_aiter; - } - if (getter == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an object with " - "__aiter__ method, got %.100s", - type->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(obj); - goto pop_1_error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - iter_o = (*getter)(obj_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(obj); - if (iter_o == NULL) goto pop_1_error; - if (Py_TYPE(iter_o)->tp_as_async == NULL || - Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' received an object from __aiter__ " - "that does not implement __anext__: %.100s", - Py_TYPE(iter_o)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - Py_DECREF(iter_o); - goto error; - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_GET_ANEXT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_GET_ANEXT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ANEXT); - _PyStackRef aiter; - _PyStackRef awaitable; - aiter = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (awaitable_o == NULL) { - goto error; - } - awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); - stack_pointer[0] = awaitable; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_GET_AWAITABLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_GET_AWAITABLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AWAITABLE); - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(iterable); - if (iter_o == NULL) goto pop_1_error; - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_GET_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_GET_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ITER); - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - /* before: [obj]; after [getiter(obj)] */ - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(iterable); - if (iter_o == NULL) goto pop_1_error; - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_GET_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_GET_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_LEN); - _PyStackRef obj; - _PyStackRef len; - obj = stack_pointer[-1]; - // PUSH(len(TOS)) - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) goto error; - PyObject *len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) goto error; - len = PyStackRef_FromPyObjectSteal(len_o); - stack_pointer[0] = len; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_GET_YIELD_FROM_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_GET_YIELD_FROM_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_YIELD_FROM_ITER); - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - /* before: [obj]; after [getiter(obj)] */ - PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); - if (PyCoro_CheckExact(iterable_o)) { - /* `iterable` is a coroutine */ - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { - /* and it is used in a 'yield from' expression of a - regular generator. */ - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_TypeError, - "cannot 'yield from' a coroutine object " - "in a non-coroutine generator"); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - iter = iterable; - } - else { - if (PyGen_CheckExact(iterable_o)) { - iter = iterable; - } - else { - /* `iterable` is not a generator. */ - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(iterable_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - goto error; - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - PyStackRef_CLOSE(iterable); - } - } - stack_pointer[-1] = iter; - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_IMPORT_FROM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_IMPORT_FROM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_FROM); - _PyStackRef from; - _PyStackRef res; - from = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) goto error; - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_IMPORT_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_IMPORT_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_NAME); - _PyStackRef level; - _PyStackRef fromlist; - _PyStackRef res; - fromlist = stack_pointer[-1]; - level = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportName(tstate, frame, name, - PyStackRef_AsPyObjectBorrow(fromlist), - PyStackRef_AsPyObjectBorrow(level)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(level); - PyStackRef_CLOSE(fromlist); - if (res_o == NULL) goto pop_2_error; - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef *func; - _PyStackRef *maybe_self; - _PyStackRef res; - /* Skip 3 cache entries */ - // _MAYBE_EXPAND_METHOD - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - func = &stack_pointer[-2 - oparg]; - maybe_self = &stack_pointer[-1 - oparg]; - if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); - PyStackRef_CLOSE(temp); - } - } - // _MONITOR_CALL - { - args = &stack_pointer[-oparg]; - maybe_self = &stack_pointer[-1 - oparg]; - func = &stack_pointer[-2 - oparg]; - int is_meth = !PyStackRef_IsNull(maybe_self[0]); - PyObject *function = PyStackRef_AsPyObjectBorrow(func[0]); - PyObject *arg0; - if (is_meth) { - arg0 = PyStackRef_AsPyObjectBorrow(maybe_self[0]); - } - else { - if (oparg) { - arg0 = PyStackRef_AsPyObjectBorrow(args[0]); - } - else { - arg0 = &_PyInstrumentation_MISSING; - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg0 - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; - } - // _DO_CALL - { - self_or_null = maybe_self; - callable = func; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - // Check if the call can be inlined or not - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - args, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - goto error; - } - frame->return_offset = 4 ; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(res_o); - } - } - } - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - uint32_t version = read_u32(&this_instr[2].cache); - (void)version; - int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2)); - int total_args = oparg + is_meth; - PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3)); - PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING - : PyStackRef_AsPyObjectBorrow(PEEK(total_args + 1)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(CALL_KW); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_FOR); - _PyStackRef receiver; - _PyStackRef value; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - /* Need to create a fake StopIteration error here, - * to conform to PEP 380 */ - if (PyStackRef_GenCheck(receiver)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; - } - } - PyStackRef_CLOSE(value); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_SEND); - _PyStackRef receiver; - _PyStackRef value; - _PyStackRef val; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; - } - } - val = value; - PyStackRef_CLOSE(receiver); - stack_pointer[-2] = val; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); - /* Skip 1 cache entry */ - _PyStackRef iter_stackref = TOP(); - PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next != NULL) { - PUSH(PyStackRef_FromPyObjectSteal(next)); - } - else { - if (_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - STACK_SHRINK(1); - PyStackRef_CLOSE(iter_stackref); - /* Skip END_FOR and POP_TOP */ - _Py_CODEUNIT *target = next_instr + oparg + 2; - INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_INSTRUCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_INSTRUCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); - _PyFrame_SetStackPointer(frame, stack_pointer); - int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_opcode < 0) goto error; - next_instr = this_instr; - if (_PyOpcode_Caches[next_opcode]) { - PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); - } - assert(next_opcode > 0 && next_opcode < 256); - opcode = next_opcode; - DISPATCH_GOTO(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); - /* Skip 1 cache entry */ - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - } - } - // _MONITOR_JUMP_BACKWARD - { - INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); - } - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_LINE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_LINE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_LINE); - int original_opcode = 0; - if (tstate->tracing) { - PyCodeObject *code = _PyFrame_GetCode(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - original_opcode = code->_co_monitoring->lines[(int)(this_instr - _PyFrame_GetBytecode(frame))].original_opcode; - stack_pointer = _PyFrame_GetStackPointer(frame); - next_instr = this_instr; - } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - original_opcode = _Py_call_instrumentation_line( - tstate, frame, this_instr, prev_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (original_opcode < 0) { - next_instr = this_instr+1; - goto error; - } - next_instr = frame->instr_ptr; - if (next_instr != this_instr) { - DISPATCH(); - } - } - if (_PyOpcode_Caches[original_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - /* Prevent the underlying instruction from specializing - * and overwriting the instrumentation. */ - PAUSE_ADAPTIVE_COUNTER(cache->counter); - } - opcode = original_opcode; - DISPATCH_GOTO(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); - /* Skip 1 cache entry */ - // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we - // don't want to specialize instrumented instructions - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); - INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); - /* Skip 1 cache entry */ - _PyStackRef cond = POP(); - assert(PyStackRef_BoolCheck(cond)); - int jump = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); - /* Skip 1 cache entry */ - _PyStackRef value_stackref = POP(); - int jump = PyStackRef_IsNone(value_stackref); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - else { - PyStackRef_CLOSE(value_stackref); - } - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); - /* Skip 1 cache entry */ - _PyStackRef value_stackref = POP(); - int jump = !PyStackRef_IsNone(value_stackref); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - PyStackRef_CLOSE(value_stackref); - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); - /* Skip 1 cache entry */ - _PyStackRef cond = POP(); - assert(PyStackRef_BoolCheck(cond)); - int jump = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RESUME); - // _LOAD_BYTECODE - { - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) goto error; - _PyFrame_SetStackPointer(frame, stack_pointer); - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; - frame->instr_ptr = bytecode + off; - // Make sure this_instr gets reset correctley for any uops that - // follow - next_instr = frame->instr_ptr; - DISPATCH(); - } - #endif - } - // _MAYBE_INSTRUMENT - { - if (tstate->tracing == 0) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; - } - next_instr = this_instr; - DISPATCH(); - } - } - } - // _CHECK_PERIODIC_IF_NOT_YIELD_FROM - { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - } - } - } - // _MONITOR_RESUME - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; - if (frame->instr_ptr != this_instr) { - /* Instrumentation has jumped */ - next_instr = frame->instr_ptr; - } - } - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); - _PyStackRef val; - _PyStackRef retval; - _PyStackRef res; - // _RETURN_VALUE_EVENT - { - val = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; - } - // _RETURN_VALUE - { - retval = val; - #if TIER_ONE - assert(frame != entry_frame); - #endif - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(EMPTY()); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(frame->return_offset); - res = temp; - LLTRACE_RESUME_FRAME(); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); - _PyStackRef val; - _PyStackRef retval; - _PyStackRef value; - // _YIELD_VALUE_EVENT - { - val = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; - } - if (frame->instr_ptr != this_instr) { - next_instr = frame->instr_ptr; - DISPATCH(); - } - } - // _YIELD_VALUE - { - retval = val; - // NOTE: It's important that YIELD_VALUE never raises an exception! - // The compiler treats any exception raised here as a failed close() - // or throw() call. - #if TIER_ONE - assert(frame != entry_frame); - #endif - frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - value = temp; - LLTRACE_RESUME_FRAME(); - } - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INTERPRETER_EXIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_INTERPRETER_EXIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INTERPRETER_EXIT); - _PyStackRef retval; - retval = stack_pointer[-1]; - assert(frame == entry_frame); - assert(_PyFrame_IsIncomplete(frame)); - /* Restore previous frame and return. */ - tstate->current_frame = frame->previous; - assert(!_PyErr_Occurred(tstate)); - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - PyObject *result = PyStackRef_AsPyObjectSteal(retval); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - /* Not strictly necessary, but prevents warnings */ - return result; - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_IS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_IS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IS_OP); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD); - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - } - } - // _JUMP_BACKWARD - { - uint16_t the_counter = read_u16(&this_instr[1].cache); - (void)the_counter; - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); - #ifdef _Py_TIER2 - #if ENABLE_SPECIALIZATION - _Py_BackoffCounter counter = this_instr[1].counter; - if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) { - _Py_CODEUNIT *start = this_instr; - /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ - while (oparg > 255) { - oparg >>= 8; - start--; - } - _PyExecutorObject *executor; - _PyFrame_SetStackPointer(frame, stack_pointer); - int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (optimized <= 0) { - this_instr[1].counter = restart_backoff_counter(counter); - if (optimized < 0) goto error; - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - this_instr[1].counter = initial_jump_backoff_counter(); - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(tstate->previous_executor == NULL); - tstate->previous_executor = Py_None; - GOTO_TIER_TWO(executor); - } - } - else { - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - } - #endif /* ENABLE_SPECIALIZATION */ - #endif /* _Py_TIER2 */ - } - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); - /* This bytecode is used in the `yield from` or `await` loop. - * If there is an interrupt, we want it handled in the innermost - * generator or coroutine, so we deliberately do not check it here. - * (see bpo-30039). - */ - JUMPBY(-oparg); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_FORWARD); - JUMPBY(oparg); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_APPEND); - _PyStackRef list; - _PyStackRef v; - v = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), - PyStackRef_AsPyObjectSteal(v)); - if (err < 0) goto pop_1_error; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LIST_EXTEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LIST_EXTEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_EXTEND); - _PyStackRef list_st; - _PyStackRef iterable_st; - iterable_st = stack_pointer[-1]; - list_st = stack_pointer[-2 - (oparg-1)]; - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (none_val == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) - { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Clear(tstate); - _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - PyStackRef_CLOSE(iterable_st); - goto pop_1_error; - } - assert(Py_IsNone(none_val)); - PyStackRef_CLOSE(iterable_st); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR); - PREDICTED(LOAD_ATTR); - _Py_CODEUNIT* const this_instr = next_instr - 10; - (void)this_instr; - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self_or_null = PyStackRef_NULL; - // _SPECIALIZE_LOAD_ATTR - { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 8 cache entries */ - // _LOAD_ATTR - { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - PyObject *attr_o; - if (oparg & 1) { - /* Designed to work in tandem with CALL, pushes two values. */ - attr_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (is_meth) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - meth | self | arg1 | ... | argN - */ - assert(attr_o != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership - } - else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - meth | NULL | arg1 | ... | argN - */ - PyStackRef_CLOSE(owner); - if (attr_o == NULL) goto pop_1_error; - self_or_null = PyStackRef_NULL; - } - } - else { - /* Classic, pushes one value. */ - _PyFrame_SetStackPointer(frame, stack_pointer); - attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(owner); - if (attr_o == NULL) goto pop_1_error; - /* We need to define self_or_null on all paths */ - self_or_null = PyStackRef_NULL; - } - attr = PyStackRef_FromPyObjectSteal(attr_o); - } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = self_or_null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _CHECK_ATTR_CLASS - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); - assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_CLASS - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = PyStackRef_FromPyObjectNew(descr); - null = PyStackRef_NULL; - PyStackRef_CLOSE(owner); - } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _CHECK_ATTR_CLASS - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); - assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); - } - // _GUARD_TYPE_VERSION - { - uint32_t type_version = read_u32(&this_instr[4].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - // _LOAD_ATTR_CLASS - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = PyStackRef_FromPyObjectNew(descr); - null = PyStackRef_NULL; - PyStackRef_CLOSE(owner); - } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - /* Skip 1 cache entry */ - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *getattribute = read_obj(&this_instr[6].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner_o); - assert(type_version != 0); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); - assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)getattribute; - assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( - tstate, PyStackRef_FromPyObjectNew(f), 2, frame); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); - new_frame->localsplus[0] = owner; - new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); - frame->return_offset = 10 ; - DISPATCH_INLINED(new_frame); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - // _CHECK_MANAGED_OBJECT_HAS_VALUES - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); - } - // _LOAD_ATTR_INSTANCE_VALUE - { - uint16_t offset = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = *value_ptr; - DEOPT_IF(attr_o == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); - null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectSteal(attr_o); - PyStackRef_CLOSE(owner); - } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - // _CHECK_ATTR_METHOD_LAZY_DICT - { - uint16_t dictoffset = read_u16(&this_instr[4].cache); - char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = *(PyObject **)ptr; - /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR); - } - /* Skip 1 cache entry */ - // _LOAD_ATTR_METHOD_LAZY_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_METHOD_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); - } - // _LOAD_ATTR_METHOD_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - /* Cached method object */ - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_MODULE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - PyDictKeysObject *mod_keys; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _CHECK_ATTR_MODULE_PUSH_KEYS - { - owner = stack_pointer[-1]; - uint32_t dict_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; - assert(dict != NULL); - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR); - mod_keys = keys; - } - // _LOAD_ATTR_MODULE_FROM_KEYS - { - uint16_t index = read_u16(&this_instr[4].cache); - assert(mod_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries)); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); - // Clear mod_keys from stack in case we need to deopt - DEOPT_IF(attr_o == NULL, LOAD_ATTR); - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); - if (!increfed) { - DEOPT_IF(true, LOAD_ATTR); - } - #else - Py_INCREF(attr_o); - attr = PyStackRef_FromPyObjectSteal(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; - PyStackRef_CLOSE(owner); - } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - PyStackRef_CLOSE(owner); - attr = PyStackRef_FromPyObjectNew(descr); - } - stack_pointer[-1] = attr; - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); - } - // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - PyStackRef_CLOSE(owner); - attr = PyStackRef_FromPyObjectNew(descr); - } - stack_pointer[-1] = attr; - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -} - - -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_PROPERTY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_PROPERTY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - } - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_PROPERTY_FRAME - { - PyObject *fget = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - assert(Py_IS_TYPE(fget, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)fget; - PyCodeObject *code = (PyCodeObject *)f->func_code; - DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR); - DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR); - DEOPT_IF(code->co_argcount != 1, LOAD_ATTR); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); - new_frame->localsplus[0] = owner; - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer += -2 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Need to sync the stack since we exit with DISPATCH_INLINED. + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + CEVAL_GOTO(error); + } + assert( 1 == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); } - assert(STACK_LEVEL() == 0); + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + stack_pointer[-1 - (oparg & 1)] = callargs_st; + if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(kwargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(callargs_st); + PyStackRef_CLOSE(func_st); + if (result_o == NULL) { + stack_pointer += -3 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + result = PyStackRef_FromPyObjectSteal(result_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3 - (oparg & 1)] = result; + stack_pointer += -2 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 2 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } -#endif - DISPATCH(); } + stack_pointer[-3 - (oparg & 1)] = result; + stack_pointer += -2 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_INTRINSIC_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_INTRINSIC_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_SLOT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - // _LOAD_ATTR_SLOT - { - uint16_t index = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - char *addr = (char *)owner_o + index; - PyObject *attr_o = *(PyObject **)addr; - DEOPT_IF(attr_o == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectNew(attr_o); - PyStackRef_CLOSE(owner); - } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_1); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(oparg <= MAX_INTRINSIC_1); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) CEVAL_GOTO(pop_1_error); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-1] = res; DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_INTRINSIC_2(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_INTRINSIC_2(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_2); + _PyStackRef value2_st; + _PyStackRef value1_st; + _PyStackRef res; + value1_st = stack_pointer[-1]; + value2_st = stack_pointer[-2]; + assert(oparg <= MAX_INTRINSIC_2); + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value2_st); + PyStackRef_CLOSE(value1_st); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_ISINSTANCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_ISINSTANCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ISINSTANCE); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - // _CHECK_ATTR_WITH_HINT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, LOAD_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* isinstance(o, o2) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; } - // _LOAD_ATTR_WITH_HINT - { - uint16_t hint = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject *attr_o; - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), LOAD_ATTR); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); - attr_o = ep->me_value; - DEOPT_IF(attr_o == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); - attr = PyStackRef_FromPyObjectSteal(attr_o); - null = PyStackRef_NULL; - PyStackRef_CLOSE(owner); + DEOPT_IF(total_args != 2, CALL); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL); + STAT_INC(CALL, hit); + _PyStackRef cls_stackref = args[1]; + _PyStackRef inst_stackref = args[0]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (retval < 0) { + CEVAL_GOTO(error); } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); + res = retval ? PyStackRef_True : PyStackRef_False; + assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(inst_stackref); + PyStackRef_CLOSE(cls_stackref); + PyStackRef_CLOSE(callable[0]); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_BUILD_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_BUILD_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_BUILD_CLASS); - _PyStackRef bc; - PyObject *bc_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) goto error; - if (bc_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_NameError, - "__build_class__ not found"); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - bc = PyStackRef_FromPyObjectSteal(bc_o); - stack_pointer[0] = bc; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); + next_instr += 4; + INSTRUCTION_STATS(CALL_KW); + PREDICTED(CALL_KW); + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef kwnames_in; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef kwnames_out; + _PyStackRef res; + // _SPECIALIZE_CALL_KW + { + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CALL_KW); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + kwnames_in = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + func = &stack_pointer[-3 - oparg]; + maybe_self = &stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); } + kwnames_out = kwnames_in; } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _DO_CALL_KW { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); + kwnames = kwnames_out; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-1] = kwnames; _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(kwnames); + // Sync stack explicitly since we leave using DISPATCH_INLINED(). + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + CEVAL_GOTO(error); + } + assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4 ; + DISPATCH_INLINED(new_frame); } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); + } } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; + stack_pointer[-1] = kwnames; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(res_o); + } } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; + PyStackRef_CLOSE(kwnames); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + if (res_o == NULL) { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } -#endif - DISPATCH(); + res = PyStackRef_FromPyObjectSteal(res_o); } + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_COMMON_CONSTANT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_KW_BOUND_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_COMMON_CONSTANT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_KW_BOUND_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); - _PyStackRef value; - // Keep in sync with _common_constants in opcode.py - // If we ever have more than two constants, use a lookup table - PyObject *val; - if (oparg == CONSTANT_ASSERTIONERROR) { - val = PyExc_AssertionError; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef kwnames; + _PyStackRef *method; + _PyStackRef *self; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL_KW); } - else { - assert(oparg == CONSTANT_NOTIMPLEMENTEDERROR); - val = PyExc_NotImplementedError; + // _CHECK_METHOD_VERSION_KW + { + null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW); + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + DEOPT_IF(!PyFunction_Check(func), CALL_KW); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW); } - value = PyStackRef_FromPyObjectImmortal(val); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); + // _EXPAND_METHOD_KW + { + method = &stack_pointer[-3 - oparg]; + self = &stack_pointer[-2 - oparg]; + _PyStackRef callable_s = callable[0]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable_s); + assert(PyStackRef_IsNull(null[0])); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(method[0])); + PyStackRef_CLOSE(callable_s); } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); + // flush + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(kwnames); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + CEVAL_GOTO(error); } + new_frame = temp; } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _SAVE_RETURN_OFFSET { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_CONST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_KW_NON_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_CONST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_KW_NON_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST); - PREDICTED(LOAD_CONST); - _PyStackRef value; - value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_NON_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef kwnames; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE_KW + { + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(PyFunction_Check(callable_o), CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _CALL_KW_NON_PY { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(kwnames); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 2 + oparg; + assert(WITHIN_STACK_BOUNDS()); } -#endif - DISPATCH(); } + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_CONST_IMMORTAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_KW_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_CONST_IMMORTAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_KW_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST_IMMORTAL); - static_assert(0 == 0, "incorrect cache size"); - _PyStackRef value; - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - assert(_Py_IsImmortal(obj)); - value = PyStackRef_FromPyObjectImmortal(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef kwnames; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL_KW); } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } + // _CHECK_FUNCTION_VERSION_KW + { + callable = &stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL_KW); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _PY_FRAME_KW { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(kwnames); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + CEVAL_GOTO(error); } + new_frame = temp; + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LEN); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* len(o) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable_o != interp->callable_cache.len, CALL); + STAT_INC(CALL, hit); + _PyStackRef arg_stackref = args[0]; + PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (len_i < 0) { + CEVAL_GOTO(error); + } + PyObject *res_o = PyLong_FromSsize_t(len_i); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + CEVAL_GOTO(error); } + PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(arg_stackref); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } +} -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LIST_APPEND); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self; + _PyStackRef arg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + arg = stack_pointer[-1]; + self = stack_pointer[-2]; + callable = stack_pointer[-3]; + assert(oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL); + assert(self_o != NULL); + DEOPT_IF(!PyList_Check(self_o), CALL); + DEOPT_IF(!LOCK_OBJECT(self_o), CALL); + STAT_INC(CALL, hit); + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); + PyStackRef_CLOSE(self); + PyStackRef_CLOSE(callable); + if (err) CEVAL_GOTO(pop_3_error); + #if TIER_ONE + // Skip the following POP_TOP. This is done here in tier one, and + // during trace projection in tier two: + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); + #endif + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_DEREF); - _PyStackRef value; - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *value_o = PyCell_GetRef(cell); - if (value_o == NULL) { + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + /* Builtin METH_FASTCALL methods, without keywords */ + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + int nargs = total_args - 1; + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); + } + } _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + PyCFunctionFast cfunc = + (PyCFunctionFast)(void(*)(void))meth->ml_meth; + PyObject *res_o = cfunc(self, (args_o + 1), nargs); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Clear the stack of the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); } - value = PyStackRef_FromPyObjectSteal(value_o); - stack_pointer[0] = value; - stack_pointer += 1; + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + PyTypeObject *d_type = method->d_common.d_type; + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + STAT_INC(CALL, hit); + int nargs = total_args - 1; + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } -#endif - DISPATCH(); } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST); - _PyStackRef value; - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_NOARGS + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + assert(oparg == 0 || oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } + res = PyStackRef_FromPyObjectSteal(res_o); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _CHECK_PERIODIC { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_O + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_O, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + _PyStackRef arg_stackref = args[1]; + _PyStackRef self_stackref = args[0]; + DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, + PyStackRef_AsPyObjectBorrow(self_stackref), + PyStackRef_AsPyObjectBorrow(arg_stackref)); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(arg_stackref); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_FAST_AND_CLEAR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_NON_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_FAST_AND_CLEAR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_NON_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); - _PyStackRef value; - value = GETLOCAL(oparg); - // do not use SETLOCAL here, it decrefs the old value - GETLOCAL(oparg) = PyStackRef_NULL; - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } + next_instr += 4; + INSTRUCTION_STATS(CALL_NON_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE + { + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(PyFunction_Check(callable_o), CALL); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _CALL_NON_PY_GENERAL { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } -#endif - DISPATCH(); + res = PyStackRef_FromPyObjectSteal(res_o); } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_FAST_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_PY_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_FAST_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_PY_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_CHECK); - _PyStackRef value; - _PyStackRef value_s = GETLOCAL(oparg); - if (PyStackRef_IsNull(value_s)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); } - value = PyStackRef_DUP(value_s); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); + // _CHECK_FUNCTION_VERSION + { + callable = &stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL); } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _CHECK_STACK_SPACE { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null[0]); + STAT_INC(CALL, hit); + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null[0]; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } -#endif - DISPATCH(); } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); - _PyStackRef value1; - _PyStackRef value2; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - value1 = PyStackRef_DUP(GETLOCAL(oparg1)); - value2 = PyStackRef_DUP(GETLOCAL(oparg2)); - stack_pointer[0] = value1; - stack_pointer[1] = value2; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } + // _CHECK_FUNCTION_VERSION + { + callable = &stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _PY_FRAME_GENERAL { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + // The frame has stolen all the arguments from the stack. + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + CEVAL_GOTO(error); } -#endif - DISPATCH(); + new_frame = temp; + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_STR_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CALL_STR_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); - _PyStackRef class_dict_st; - _PyStackRef value; - class_dict_st = stack_pointer[-1]; - PyObject *value_o; - PyObject *name; - PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); - assert(class_dict); - assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); - name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - goto error; + next_instr += 4; + INSTRUCTION_STATS(CALL_STR_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_STR_1 + { + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Str(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(arg); + if (res_o == NULL) CEVAL_GOTO(pop_3_error); + res = PyStackRef_FromPyObjectSteal(res_o); } - if (!value_o) { - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - value_o = PyCell_GetRef(cell); - if (value_o == NULL) { + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } } - PyStackRef_CLOSE(class_dict_st); - value = PyStackRef_FromPyObjectSteal(value_o); - stack_pointer[-1] = value; + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_TUPLE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_TUPLE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TUPLE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_TUPLE_1 + { + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PySequence_Tuple(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(arg); + if (res_o == NULL) CEVAL_GOTO(pop_3_error); + res = PyStackRef_FromPyObjectSteal(res_o); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _CHECK_PERIODIC { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } + } + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_TYPE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CALL_TYPE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TYPE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL); + STAT_INC(CALL, hit); + res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); + PyStackRef_CLOSE(arg); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CHECK_EG_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CHECK_EG_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -22777,676 +4036,682 @@ _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(_PyInterpreterFrame *frame, _PyStackRef *st { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); - _PyStackRef mod_or_class_dict; - _PyStackRef v; - mod_or_class_dict = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *v_o; + INSTRUCTION_STATS(CHECK_EG_MATCH); + _PyStackRef exc_value_st; + _PyStackRef match_type_st; + _PyStackRef rest; + _PyStackRef match; + match_type_st = stack_pointer[-1]; + exc_value_st = stack_pointer[-2]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); + int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(mod_or_class_dict); - if (err < 0) goto pop_1_error; - if (v_o == NULL) { - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - goto error; - } - } - else { - /* Slow-path if globals or builtins is not a dict */ - /* namespace 1: globals */ - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) goto error; - if (v_o == NULL) { - /* namespace 2: builtins */ - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) goto error; - if (v_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - } - } - stack_pointer += 1; + if (err < 0) { + PyStackRef_CLOSE(exc_value_st); + PyStackRef_CLOSE(match_type_st); + CEVAL_GOTO(pop_2_error); + } + PyObject *match_o = NULL; + PyObject *rest_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, + &match_o, &rest_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(exc_value_st); + PyStackRef_CLOSE(match_type_st); + if (res < 0) CEVAL_GOTO(pop_2_error); + assert((match_o == NULL) == (rest_o == NULL)); + if (match_o == NULL) CEVAL_GOTO(pop_2_error); + if (!Py_IsNone(match_o)) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyErr_SetHandledException(match_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); } - v = PyStackRef_FromPyObjectSteal(v_o); - stack_pointer[-1] = v; + rest = PyStackRef_FromPyObjectSteal(rest_o); + match = PyStackRef_FromPyObjectSteal(match_o); + stack_pointer[-2] = rest; + stack_pointer[-1] = match; DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CHECK_EXC_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CHECK_EXC_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EXC_MATCH); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyExceptionInstance_Check(left_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptTypeValid(tstate, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + PyStackRef_CLOSE(right); + CEVAL_GOTO(pop_1_error); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyErr_GivenExceptionMatches(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(right); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[-1] = b; + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CLEANUP_THROW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CLEANUP_THROW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(CLEANUP_THROW); + _PyStackRef sub_iter_st; + _PyStackRef last_sent_val_st; + _PyStackRef exc_value_st; + _PyStackRef none; + _PyStackRef value; + exc_value_st = stack_pointer[-1]; + last_sent_val_st = stack_pointer[-2]; + sub_iter_st = stack_pointer[-3]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + // assert(throwflag); + assert(exc_value && PyExceptionInstance_Check(exc_value)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + none = PyStackRef_None; + value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); + PyStackRef_CLOSE(sub_iter_st); + PyStackRef_CLOSE(last_sent_val_st); + PyStackRef_CLOSE(exc_value_st); } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + none = PyStackRef_NULL; + value = PyStackRef_NULL; + CEVAL_GOTO(exception_unwind); + } + stack_pointer[-3] = none; + stack_pointer[-2] = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_COMPARE_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_COMPARE_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL); - PREDICTED(LOAD_GLOBAL); - _Py_CODEUNIT* const this_instr = next_instr - 5; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP); + PREDICTED(COMPARE_OP); + _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - _PyStackRef *res; - _PyStackRef null = PyStackRef_NULL; - // _SPECIALIZE_LOAD_GLOBAL + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _SPECIALIZE_COMPARE_OP { + right = stack_pointer[-1]; + left = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION_FT + #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + _Py_Specialize_CompareOp(left, right, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } - OPCODE_DEFERRED_INC(LOAD_GLOBAL); + OPCODE_DEFERRED_INC(COMPARE_OP); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ + #endif /* ENABLE_SPECIALIZATION */ } - /* Skip 1 cache entry */ - /* Skip 1 cache entry */ - /* Skip 1 cache entry */ - // _LOAD_GLOBAL + // _COMPARE_OP { - res = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert((oparg >> 5) <= Py_GE); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*res)) goto error; - null = PyStackRef_NULL; + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); + if (oparg & 16) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res_bool = PyObject_IsTrue(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(res_o); + if (res_bool < 0) CEVAL_GOTO(error); + res = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + } } - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_COMPARE_OP_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_COMPARE_OP_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_FLOAT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + /* Skip 1 cache entry */ + // _COMPARE_OP_FLOAT { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); + // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg + int sign_ish = COMPARISON_BIT(dleft, dright); + PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + // It's always a bool, so we don't care about oparg & 16. + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_COMPARE_OP_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_COMPARE_OP_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_INT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_INT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP); + } + /* Skip 1 cache entry */ + // _COMPARE_OP_INT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && + _PyLong_DigitCount((PyLongObject *)right_o) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); + // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg + int sign_ish = COMPARISON_BIT(ileft, iright); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + // It's always a bool, so we don't care about oparg & 16. + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_COMPARE_OP_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_COMPARE_OP_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_STR); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_UNICODE + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP); } + /* Skip 1 cache entry */ + // _COMPARE_OP_STR + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left_o, right_o); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; + // It's always a bool, so we don't care about oparg & 16. + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_GLOBAL_BUILTIN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CONTAINS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_GLOBAL_BUILTIN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_CONTAINS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - PyDictKeysObject *builtins_keys; - _PyStackRef res; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_GLOBALS_VERSION - { - uint16_t version = read_u16(&this_instr[2].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(keys)); - } - // _GUARD_BUILTINS_VERSION_PUSH_KEYS + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP); + PREDICTED(CONTAINS_OP); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + // _SPECIALIZE_CONTAINS_OP { - uint16_t version = read_u16(&this_instr[3].cache); - PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); - builtins_keys = keys; - assert(DK_IS_UNICODE(builtins_keys)); + right = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ContainsOp(right, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CONTAINS_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } - // _LOAD_GLOBAL_BUILTINS_FROM_KEYS + // _CONTAINS_OP { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL); - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL); - #else - Py_INCREF(res_o); - res = PyStackRef_FromPyObjectSteal(res_o); - #endif - STAT_INC(LOAD_GLOBAL, hit); - null = PyStackRef_NULL; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PySequence_Contains(right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) CEVAL_GOTO(pop_2_error); + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer[-2] = b; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CONTAINS_OP_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CONTAINS_OP_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_DICT); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + /* Skip 1 cache entry */ + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP); + STAT_INC(CONTAINS_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyDict_Contains(right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) CEVAL_GOTO(pop_2_error); + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CONTAINS_OP_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CONTAINS_OP_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_SET); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + /* Skip 1 cache entry */ + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP); + STAT_INC(CONTAINS_OP, hit); + // Note: both set and frozenset use the same seq_contains method! + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PySet_Contains((PySetObject *)right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) CEVAL_GOTO(pop_2_error); + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} + - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CONVERT_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_CONVERT_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONVERT_VALUE); + _PyStackRef value; + _PyStackRef result; + value = stack_pointer[-1]; + conversion_func conv_fn; + assert(oparg >= FVC_STR && oparg <= FVC_ASCII); + conv_fn = _PyEval_ConversionFuncs[oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (result_o == NULL) CEVAL_GOTO(pop_1_error); + result = PyStackRef_FromPyObjectSteal(result_o); + stack_pointer[-1] = result; + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_GLOBAL_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_COPY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_GLOBAL_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_COPY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - PyDictKeysObject *globals_keys; - _PyStackRef res; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_GLOBALS_VERSION_PUSH_KEYS - { - uint16_t version = read_u16(&this_instr[2].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); - globals_keys = keys; - assert(DK_IS_UNICODE(globals_keys)); - } - /* Skip 1 cache entry */ - // _LOAD_GLOBAL_MODULE_FROM_KEYS - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL); - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL); - #else - Py_INCREF(res_o); - res = PyStackRef_FromPyObjectSteal(res_o); - #endif - STAT_INC(LOAD_GLOBAL, hit); - null = PyStackRef_NULL; - } - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY); + _PyStackRef bottom; + _PyStackRef top; + bottom = stack_pointer[-1 - (oparg-1)]; + assert(oparg > 0); + top = PyStackRef_DUP(bottom); + stack_pointer[0] = top; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_COPY_FREE_VARS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_COPY_FREE_VARS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY_FREE_VARS); + /* Copy closure variables to free variables */ + PyCodeObject *co = _PyFrame_GetCode(frame); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; + assert(oparg == co->co_nfreevars); + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_DELETE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_DELETE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_ATTR); + _PyStackRef owner; + owner = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(owner); + if (err) CEVAL_GOTO(pop_1_error); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } +} -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_DELETE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_DELETE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_DEREF); + PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + // Can't use ERROR_IF here. + // Fortunately we don't need its superpower. + PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); + if (oldobj == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); + } + Py_DECREF(oldobj); + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_LOCALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_DELETE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_LOCALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_DELETE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -23454,136 +4719,30 @@ _TAIL_CALL_LOAD_LOCALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(LOAD_LOCALS); - _PyStackRef locals; - PyObject *l = LOCALS(); - if (l == NULL) { + INSTRUCTION_STATS(DELETE_FAST); + _PyStackRef v = GETLOCAL(oparg); + if (PyStackRef_IsNull(v)) { _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + CEVAL_GOTO(error); } - locals = PyStackRef_FromPyObjectNew(l); - stack_pointer[0] = locals; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + SETLOCAL(oparg, PyStackRef_NULL); DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_DELETE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_DELETE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -23591,133 +4750,76 @@ _TAIL_CALL_LOAD_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(LOAD_NAME); - _PyStackRef v; + INSTRUCTION_STATS(DELETE_GLOBAL); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + int err = PyDict_Pop(GLOBALS(), name, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) goto error; - v = PyStackRef_FromPyObjectSteal(v_o); - stack_pointer[0] = v; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); + // Can't use ERROR_IF here. + if (err < 0) { + CEVAL_GOTO(error); } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_DELETE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_DELETE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_NAME); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals when deleting %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); } + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_DelItem(ns, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Can't use ERROR_IF here. + if (err != 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, + name); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); + } + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_SMALL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_DELETE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_SMALL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_DELETE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -23725,130 +4827,33 @@ _TAIL_CALL_LOAD_SMALL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(LOAD_SMALL_INT); - _PyStackRef value; - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectImmortal(obj); - stack_pointer[0] = value; - stack_pointer += 1; + INSTRUCTION_STATS(DELETE_SUBSCR); + _PyStackRef container; + _PyStackRef sub; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + /* del container[sub] */ + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), + PyStackRef_AsPyObjectBorrow(sub)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (err) CEVAL_GOTO(pop_2_error); + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_SPECIAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_DICT_MERGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_SPECIAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_DICT_MERGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -23856,956 +4861,684 @@ _TAIL_CALL_LOAD_SPECIAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(LOAD_SPECIAL); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self_or_null; - owner = stack_pointer[-1]; - assert(oparg <= SPECIAL_MAX); - PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); - PyObject *name = _Py_SpecialMethods[oparg].name; - PyObject *self_or_null_o; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = _PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - _Py_SpecialMethods[oparg].error, - Py_TYPE(owner_o)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - goto error; - } - attr = PyStackRef_FromPyObjectSteal(attr_o); - self_or_null = self_or_null_o == NULL ? - PyStackRef_NULL : PyStackRef_FromPyObjectSteal(self_or_null_o); - stack_pointer[0] = attr; - stack_pointer[1] = self_or_null; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; + INSTRUCTION_STATS(DICT_MERGE); + _PyStackRef callable; + _PyStackRef dict; + _PyStackRef update; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_MergeEx(dict_o, update_o, 2); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatKwargsError(tstate, callable_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(update); + CEVAL_GOTO(pop_1_error); + } + PyStackRef_CLOSE(update); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_DICT_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_DICT_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR); - PREDICTED(LOAD_SUPER_ATTR); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - // _SPECIALIZE_LOAD_SUPER_ATTR - { - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - int load_method = oparg & 1; - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _LOAD_SUPER_ATTR - { - self_st = stack_pointer[-1]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - goto pop_3_error; - } - } - // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = {class, self}; + next_instr += 1; + INSTRUCTION_STATS(DICT_UPDATE); + _PyStackRef dict; + _PyStackRef update; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_Update(dict_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(super); - } - } + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object is not a mapping", + Py_TYPE(update_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); } - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - if (super == NULL) goto pop_3_error; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(super, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - Py_DECREF(super); - if (attr_o == NULL) goto error; - attr = PyStackRef_FromPyObjectSteal(attr_o); - null = PyStackRef_NULL; + PyStackRef_CLOSE(update); + CEVAL_GOTO(pop_1_error); } - stack_pointer[0] = attr; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + PyStackRef_CLOSE(update); + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_END_ASYNC_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_END_ASYNC_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(END_ASYNC_FOR); + _PyStackRef awaitable_st; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; + awaitable_st = stack_pointer[-2]; + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(exc && PyExceptionInstance_Check(exc)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + PyStackRef_CLOSE(awaitable_st); + PyStackRef_CLOSE(exc_st); + } + else { + Py_INCREF(exc); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(exception_unwind); } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } +} -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_FOR); + _PyStackRef value; + value = stack_pointer[-1]; + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr_st; - /* Skip 1 cache entry */ - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - if (attr == NULL) goto pop_3_error; - attr_st = PyStackRef_FromPyObjectSteal(attr); - stack_pointer[-3] = attr_st; - stack_pointer += -2; + next_instr += 1; + INSTRUCTION_STATS(END_SEND); + _PyStackRef receiver; + _PyStackRef value; + _PyStackRef val; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + (void)receiver; + val = value; + PyStackRef_CLOSE(receiver); + stack_pointer[-2] = val; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_ENTER_EXECUTOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_ENTER_EXECUTOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(ENTER_EXECUTOR); + #ifdef _Py_TIER2 + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; + assert(executor->vm_data.index == INSTR_OFFSET() - 1); + assert(executor->vm_data.code == code); + assert(executor->vm_data.valid); + assert(tstate->previous_executor == NULL); + /* If the eval breaker is set then stay in tier 1. + * This avoids any potentially infinite loops + * involving _RESUME_CHECK */ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + opcode = executor->vm_data.opcode; + oparg = (oparg & ~255) | executor->vm_data.oparg; + next_instr = this_instr; + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } + DISPATCH_GOTO(); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); + tstate->previous_executor = Py_None; + Py_INCREF(executor); + GOTO_TIER_TWO(executor); + #else + Py_FatalError("ENTER_EXECUTOR is not supported in this build"); + #endif /* _Py_TIER2 */ + DISPATCH(); + } +} - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_EXIT_INIT_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_EXIT_INIT_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXIT_INIT_CHECK); + _PyStackRef should_be_none; + should_be_none = stack_pointer[-1]; + assert(STACK_LEVEL() == 2); + if (!PyStackRef_IsNone(should_be_none)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyErr_Format(PyExc_TypeError, + "__init__() should return None, not '%.200s'", + Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_EXTENDED_ARG(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_EXTENDED_ARG(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXTENDED_ARG); + assert(oparg); + opcode = next_instr->op.code; + oparg = oparg << 8 | next_instr->op.arg; + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); } +} -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_FORMAT_SIMPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_FORMAT_SIMPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_SIMPLE); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + /* If value is a unicode object, then we know the result + * of format(value) is value itself. */ + if (!PyUnicode_CheckExact(value_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(value_o, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) CEVAL_GOTO(pop_1_error); + res = PyStackRef_FromPyObjectSteal(res_o); + } + else { + res = value; + } + stack_pointer[-1] = res; + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_FORMAT_WITH_SPEC(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_FORMAT_WITH_SPEC(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef self_or_null; - /* Skip 1 cache entry */ - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - PyTypeObject *cls = (PyTypeObject *)class; - int method_found = 0; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_WITH_SPEC); + _PyStackRef value; + _PyStackRef fmt_spec; + _PyStackRef res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - if (attr_o == NULL) { - PyStackRef_CLOSE(self_st); - goto pop_3_error; - } - if (method_found) { - self_or_null = self_st; // transfer ownership - } else { - PyStackRef_CLOSE(self_st); - self_or_null = PyStackRef_NULL; - } - attr = PyStackRef_FromPyObjectSteal(attr_o); - stack_pointer[-3] = attr; - stack_pointer[-2] = self_or_null; + PyStackRef_CLOSE(value); + PyStackRef_CLOSE(fmt_spec); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER); + PREDICTED(FOR_ITER); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef iter; + _PyStackRef next; + // _SPECIALIZE_FOR_ITER { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); + iter = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); + _Py_Specialize_ForIter(iter, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; + OPCODE_DEFERRED_INC(FOR_ITER); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + // _FOR_ITER + { + /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_o == NULL) { + if (_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!matches) { + CEVAL_GOTO(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + PyStackRef_CLOSE(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */ + JUMPBY(oparg + 2); + DISPATCH(); } + next = PyStackRef_FromPyObjectSteal(next_o); + // Common case: no jump, leave it to the code generator + } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_FOR_ITER_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_FOR_ITER_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_GEN); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyInterpreterFrame *gen_frame; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); } + // _FOR_ITER_GEN_FRAME + { + iter = stack_pointer[-1]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); + STAT_INC(FOR_ITER, hit); + gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, PyStackRef_None); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + gen_frame->previous = frame; + // oparg is the return offset from the next instruction. + frame->return_offset = (uint16_t)( 2 + oparg); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_MAKE_CELL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_FOR_ITER_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_MAKE_CELL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_FOR_ITER_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_CELL); - // "initial" is probably NULL but not if it's an arg (or set - // via the f_locals proxy before MAKE_CELL has run). - PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *cell = PyCell_New(initial); - if (cell == NULL) { - goto error; - } - SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell)); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_LIST); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_LIST + { + iter = stack_pointer[-1]; + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _ITER_JUMP_LIST { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); + STAT_INC(FOR_ITER, hit); + PyListObject *seq = it->it_seq; + if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { + it->it_index = -1; + #ifndef Py_GIL_DISABLED + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + #endif + PyStackRef_CLOSE(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ + JUMPBY(oparg + 2); + DISPATCH(); } -#endif - DISPATCH(); } + // _ITER_NEXT_LIST + { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); + PyListObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyList_GET_SIZE(seq)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_MAKE_FUNCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_FOR_ITER_RANGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_MAKE_FUNCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_FOR_ITER_RANGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_FUNCTION); - _PyStackRef codeobj_st; - _PyStackRef func; - codeobj_st = stack_pointer[-1]; - PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(codeobj_st); - if (func_obj == NULL) goto pop_1_error; - _PyFunction_SetVersion( - func_obj, ((PyCodeObject *)codeobj)->co_version); - func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); - stack_pointer[-1] = func; - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_RANGE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_RANGE + { + iter = stack_pointer[-1]; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); + // _ITER_JUMP_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + STAT_INC(FOR_ITER, hit); + if (r->len <= 0) { + STACK_SHRINK(1); + PyStackRef_CLOSE(iter); + // Jump over END_FOR and POP_TOP instructions. + JUMPBY(oparg + 2); + DISPATCH(); } } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _ITER_NEXT_RANGE { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + assert(r->len > 0); + long value = r->start; + r->start = value + r->step; + r->len--; + PyObject *res = PyLong_FromLong(value); + if (res == NULL) CEVAL_GOTO(error); + next = PyStackRef_FromPyObjectSteal(res); + } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_FOR_ITER_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_FOR_ITER_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_TUPLE + { + iter = stack_pointer[-1]; + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER); } + // _ITER_JUMP_TUPLE + { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); + STAT_INC(FOR_ITER, hit); + PyTupleObject *seq = it->it_seq; + if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); + } + PyStackRef_CLOSE(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ + JUMPBY(oparg + 2); + DISPATCH(); + } + } + // _ITER_NEXT_TUPLE + { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); + PyTupleObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyTuple_GET_SIZE(seq)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); + } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_MAP_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_GET_AITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_MAP_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_GET_AITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -24813,143 +5546,92 @@ _TAIL_CALL_MAP_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(MAP_ADD); - _PyStackRef dict_st; - _PyStackRef key; - _PyStackRef value; - value = stack_pointer[-1]; - key = stack_pointer[-2]; - dict_st = stack_pointer[-3 - (oparg - 1)]; - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); - /* dict[key] = value */ - // Do not DECREF INPUTS because the function steals the references + INSTRUCTION_STATS(GET_AITER); + _PyStackRef obj; + _PyStackRef iter; + obj = stack_pointer[-1]; + unaryfunc getter = NULL; + PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); + PyObject *iter_o; + PyTypeObject *type = Py_TYPE(obj_o); + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; + } + if (getter == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an object with " + "__aiter__ method, got %.100s", + type->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(obj); + CEVAL_GOTO(pop_1_error); + } _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2( - (PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(key), - PyStackRef_AsPyObjectSteal(value) - ); + iter_o = (*getter)(obj_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto pop_2_error; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } + PyStackRef_CLOSE(obj); + if (iter_o == NULL) CEVAL_GOTO(pop_1_error); + if (Py_TYPE(iter_o)->tp_as_async == NULL || + Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' received an object from __aiter__ " + "that does not implement __anext__: %.100s", + Py_TYPE(iter_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(iter_o); + CEVAL_GOTO(error); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[-1] = iter; + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_GET_ANEXT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_GET_ANEXT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ANEXT); + _PyStackRef aiter; + _PyStackRef awaitable; + aiter = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (awaitable_o == NULL) { + CEVAL_GOTO(error); } + awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); + stack_pointer[0] = awaitable; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_MATCH_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_GET_AWAITABLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_MATCH_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_GET_AWAITABLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -24957,154 +5639,29 @@ _TAIL_CALL_MATCH_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(MATCH_CLASS); - _PyStackRef subject; - _PyStackRef type; - _PyStackRef names; - _PyStackRef attrs; - names = stack_pointer[-1]; - type = stack_pointer[-2]; - subject = stack_pointer[-3]; - // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or - // None on failure. - assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + INSTRUCTION_STATS(GET_AWAITABLE); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attrs_o = _PyEval_MatchClass(tstate, - PyStackRef_AsPyObjectBorrow(subject), - PyStackRef_AsPyObjectBorrow(type), oparg, - PyStackRef_AsPyObjectBorrow(names)); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(subject); - PyStackRef_CLOSE(type); - PyStackRef_CLOSE(names); - if (attrs_o) { - assert(PyTuple_CheckExact(attrs_o)); // Success! - attrs = PyStackRef_FromPyObjectSteal(attrs_o); - } - else { - if (_PyErr_Occurred(tstate)) goto pop_3_error; - // Error! - attrs = PyStackRef_None; // Failure! - } - stack_pointer[-3] = attrs; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + PyStackRef_CLOSE(iterable); + if (iter_o == NULL) CEVAL_GOTO(pop_1_error); + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[-1] = iter; DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_MATCH_KEYS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_GET_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_MATCH_KEYS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_GET_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -25112,138 +5669,30 @@ _TAIL_CALL_MATCH_KEYS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(MATCH_KEYS); - _PyStackRef subject; - _PyStackRef keys; - _PyStackRef values_or_none; - keys = stack_pointer[-1]; - subject = stack_pointer[-2]; - // On successful match, PUSH(values). Otherwise, PUSH(None). + INSTRUCTION_STATS(GET_ITER); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + /* before: [obj]; after [getiter(obj)] */ _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, - PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (values_or_none_o == NULL) goto error; - values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); - stack_pointer[0] = values_or_none; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - + PyStackRef_CLOSE(iterable); + if (iter_o == NULL) CEVAL_GOTO(pop_1_error); + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[-1] = iter; + DISPATCH(); + } } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_MATCH_MAPPING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_GET_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_MATCH_MAPPING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_GET_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -25251,131 +5700,89 @@ _TAIL_CALL_MATCH_MAPPING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(MATCH_MAPPING); - _PyStackRef subject; - _PyStackRef res; - subject = stack_pointer[-1]; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; + INSTRUCTION_STATS(GET_LEN); + _PyStackRef obj; + _PyStackRef len; + obj = stack_pointer[-1]; + // PUSH(len(TOS)) + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (len_i < 0) CEVAL_GOTO(error); + PyObject *len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) CEVAL_GOTO(error); + len = PyStackRef_FromPyObjectSteal(len_o); + stack_pointer[0] = len; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_GET_YIELD_FROM_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_GET_YIELD_FROM_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_YIELD_FROM_ITER); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + /* before: [obj]; after [getiter(obj)] */ + PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); + if (PyCoro_CheckExact(iterable_o)) { + /* `iterable` is a coroutine */ + if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + /* and it is used in a 'yield from' expression of a + regular generator. */ _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; + _PyErr_SetString(tstate, PyExc_TypeError, + "cannot 'yield from' a coroutine object " + "in a non-coroutine generator"); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); + iter = iterable; + } + else { + if (PyGen_CheckExact(iterable_o)) { + iter = iterable; } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; + else { + /* `iterable` is not a generator. */ + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(iterable_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + CEVAL_GOTO(error); } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + iter = PyStackRef_FromPyObjectSteal(iter_o); + PyStackRef_CLOSE(iterable); } -#endif - DISPATCH(); } + stack_pointer[-1] = iter; + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_MATCH_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_IMPORT_FROM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_MATCH_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_IMPORT_FROM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -25383,131 +5790,31 @@ _TAIL_CALL_MATCH_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(MATCH_SEQUENCE); - _PyStackRef subject; + INSTRUCTION_STATS(IMPORT_FROM); + _PyStackRef from; _PyStackRef res; - subject = stack_pointer[-1]; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; + from = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) CEVAL_GOTO(error); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_NOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_IMPORT_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_NOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_IMPORT_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -25515,123 +5822,216 @@ _TAIL_CALL_NOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(NOP); + INSTRUCTION_STATS(IMPORT_NAME); + _PyStackRef level; + _PyStackRef fromlist; + _PyStackRef res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportName(tstate, frame, name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(level); + PyStackRef_CLOSE(fromlist); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef res; + /* Skip 3 cache entries */ + // _MAYBE_EXPAND_METHOD + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + func = &stack_pointer[-2 - oparg]; + maybe_self = &stack_pointer[-1 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); } } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _MONITOR_CALL { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); + args = &stack_pointer[-oparg]; + maybe_self = &stack_pointer[-1 - oparg]; + func = &stack_pointer[-2 - oparg]; + int is_meth = !PyStackRef_IsNull(maybe_self[0]); + PyObject *function = PyStackRef_AsPyObjectBorrow(func[0]); + PyObject *arg0; + if (is_meth) { + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self[0]); + } + else { + if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); } - assert(STACK_LEVEL() == 0); + else { + arg0 = &_PyInstrumentation_MISSING; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg0 + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) CEVAL_GOTO(error); + } + // _DO_CALL + { + self_or_null = maybe_self; + callable = func; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + CEVAL_GOTO(error); + } + frame->return_offset = 4 ; + DISPATCH_INLINED(new_frame); } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); + } } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(res_o); + } } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } -#endif - DISPATCH(); + res = PyStackRef_FromPyObjectSteal(res_o); } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -25639,257 +6039,94 @@ _TAIL_CALL_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(NOT_TAKEN); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; + INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); + GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_POP_EXCEPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_INSTRUMENTED_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_POP_EXCEPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_INSTRUMENTED_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_EXCEPT); - _PyStackRef exc_value; - exc_value = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + uint32_t version = read_u32(&this_instr[2].cache); + (void)version; + int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2)); + int total_args = oparg + is_meth; + PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3)); + PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING + : PyStackRef_AsPyObjectBorrow(PEEK(total_args + 1)); _PyFrame_SetStackPointer(frame, stack_pointer); - Py_XSETREF(exc_info->exc_value, - PyStackRef_IsNone(exc_value) - ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } + if (err) CEVAL_GOTO(error); + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + GO_TO_INSTRUCTION(CALL_KW); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_FOR); + _PyStackRef receiver; + _PyStackRef value; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + /* Need to create a fake StopIteration error here, + * to conform to PEP 380 */ + if (PyStackRef_GenCheck(receiver)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + CEVAL_GOTO(error); + } + } + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_INSTRUMENTED_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_INSTRUMENTED_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -25897,133 +6134,39 @@ _TAIL_CALL_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_poin { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_FALSE); - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_SEND); + _PyStackRef receiver; + _PyStackRef value; + _PyStackRef val; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + CEVAL_GOTO(error); + } + } + val = value; + PyStackRef_CLOSE(receiver); + stack_pointer[-2] = val; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_INSTRUMENTED_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_INSTRUMENTED_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -26032,148 +6175,82 @@ _TAIL_CALL_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_point _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NONE); - _PyStackRef value; - _PyStackRef b; - _PyStackRef cond; + INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); /* Skip 1 cache entry */ - // _IS_NONE - { - value = stack_pointer[-1]; - if (PyStackRef_IsNone(value)) { - b = PyStackRef_True; - } - else { - b = PyStackRef_False; - PyStackRef_CLOSE(value); - } - } - // _POP_JUMP_IF_TRUE - { - cond = b; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } + _PyStackRef iter_stackref = TOP(); + PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next != NULL) { + PUSH(PyStackRef_FromPyObjectSteal(next)); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); + else { + if (_PyErr_Occurred(tstate)) { _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!matches) { + CEVAL_GOTO(error); } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); } -#endif - DISPATCH(); + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + STACK_SHRINK(1); + PyStackRef_CLOSE(iter_stackref); + /* Skip END_FOR and POP_TOP */ + _Py_CODEUNIT *target = next_instr + oparg + 2; + INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH_RIGHT); } + DISPATCH(); } +} -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_INSTRUCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_INSTRUCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + _PyFrame_SetStackPointer(frame, stack_pointer); + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_opcode < 0) CEVAL_GOTO(error); + next_instr = this_instr; + if (_PyOpcode_Caches[next_opcode]) { + PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); + } + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); + } } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -26182,148 +6259,154 @@ _TAIL_CALL_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_p _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); - _PyStackRef value; - _PyStackRef b; - _PyStackRef cond; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); /* Skip 1 cache entry */ - // _IS_NONE + // _CHECK_PERIODIC { - value = stack_pointer[-1]; - if (PyStackRef_IsNone(value)) { - b = PyStackRef_True; - } - else { - b = PyStackRef_False; - PyStackRef_CLOSE(value); + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); } } - // _POP_JUMP_IF_FALSE + // _MONITOR_JUMP_BACKWARD { - cond = b; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); + DISPATCH(); + } +} - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_LINE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_LINE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_LINE); + int original_opcode = 0; + if (tstate->tracing) { + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = code->_co_monitoring->lines[(int)(this_instr - _PyFrame_GetBytecode(frame))].original_opcode; + stack_pointer = _PyFrame_GetStackPointer(frame); + next_instr = this_instr; + } else { + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = _Py_call_instrumentation_line( + tstate, frame, this_instr, prev_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = this_instr+1; + CEVAL_GOTO(error); } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); + next_instr = frame->instr_ptr; + if (next_instr != this_instr) { + DISPATCH(); } + } + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + /* Prevent the underlying instruction from specializing + * and overwriting the instrumentation. */ + PAUSE_ADAPTIVE_COUNTER(cache->counter); + } + opcode = original_opcode; + DISPATCH_GOTO(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); + /* Skip 1 cache entry */ + // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we + // don't want to specialize instrumented instructions + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } +} -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -26332,403 +6415,341 @@ _TAIL_CALL_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_point _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_TRUE); - _PyStackRef cond; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); /* Skip 1 cache entry */ - cond = stack_pointer[-1]; + _PyStackRef cond = POP(); assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + int jump = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); + /* Skip 1 cache entry */ + _PyStackRef value_stackref = POP(); + int jump = PyStackRef_IsNone(value_stackref); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } + else { + PyStackRef_CLOSE(value_stackref); + } + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); + /* Skip 1 cache entry */ + _PyStackRef value_stackref = POP(); + int jump = !PyStackRef_IsNone(value_stackref); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + PyStackRef_CLOSE(value_stackref); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_POP_TOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_POP_TOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_TOP); - _PyStackRef value; - value = stack_pointer[-1]; - PyStackRef_CLOSE(value); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); + /* Skip 1 cache entry */ + _PyStackRef cond = POP(); + assert(PyStackRef_BoolCheck(cond)); + int jump = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RESUME); + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (bytecode == NULL) CEVAL_GOTO(error); + _PyFrame_SetStackPointer(frame, stack_pointer); + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + // Make sure this_instr gets reset correctley for any uops that + // follow + next_instr = frame->instr_ptr; + DISPATCH(); } + #endif } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _MAYBE_INSTRUMENT { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); + if (tstate->tracing == 0) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + CEVAL_GOTO(error); + } + next_instr = this_instr; + DISPATCH(); } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + } + // _MONITOR_RESUME + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation( + tstate, oparg > 0, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) CEVAL_GOTO(error); + if (frame->instr_ptr != this_instr) { + /* Instrumentation has jumped */ + next_instr = frame->instr_ptr; } -#endif - DISPATCH(); } + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_PUSH_EXC_INFO(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_PUSH_EXC_INFO(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; - INSTRUCTION_STATS(PUSH_EXC_INFO); - _PyStackRef exc; - _PyStackRef prev_exc; - _PyStackRef new_exc; - exc = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - if (exc_info->exc_value != NULL) { - prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); + INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); + _PyStackRef val; + _PyStackRef retval; + _PyStackRef res; + // _RETURN_VALUE_EVENT + { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) CEVAL_GOTO(error); } - else { - prev_exc = PyStackRef_None; + // _RETURN_VALUE + { + retval = val; + #if TIER_ONE + assert(frame != entry_frame); + #endif + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(frame->return_offset); + res = temp; + LLTRACE_RESUME_FRAME(); } - assert(PyStackRef_ExceptionInstanceCheck(exc)); - exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); - new_exc = exc; - stack_pointer[-1] = prev_exc; - stack_pointer[0] = new_exc; + stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + _PyStackRef val; + _PyStackRef retval; + _PyStackRef value; + // _YIELD_VALUE_EVENT { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + CEVAL_GOTO(error); } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + if (frame->instr_ptr != this_instr) { + next_instr = frame->instr_ptr; + DISPATCH(); } -#endif - DISPATCH(); } + // _YIELD_VALUE + { + retval = val; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + #if TIER_ONE + assert(frame != entry_frame); + #endif + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + value = temp; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_PUSH_NULL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_INTERPRETER_EXIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_PUSH_NULL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_INTERPRETER_EXIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -26736,270 +6757,63 @@ _TAIL_CALL_PUSH_NULL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(PUSH_NULL); - _PyStackRef res; - res = PyStackRef_NULL; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ + INSTRUCTION_STATS(INTERPRETER_EXIT); + _PyStackRef retval; + retval = stack_pointer[-1]; + assert(frame == entry_frame); + assert(_PyFrame_IsIncomplete(frame)); + /* Restore previous frame and return. */ tstate->current_frame = frame->previous; + assert(!_PyErr_Occurred(tstate)); tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; + PyObject *result = PyStackRef_AsPyObjectSteal(retval); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + /* Not strictly necessary, but prevents warnings */ + return result; } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_RAISE_VARARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_IS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_RAISE_VARARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_IS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(RAISE_VARARGS); - _PyStackRef *args; - args = &stack_pointer[-oparg]; - assert(oparg < 3); - PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; - PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = do_raise(tstate, exc, cause); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - assert(oparg == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_reraise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto exception_unwind; - } - goto error; - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IS_OP); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_RERAISE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_RERAISE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -27007,159 +6821,70 @@ _TAIL_CALL_RERAISE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(RERAISE); - _PyStackRef *values; - _PyStackRef exc_st; - exc_st = stack_pointer[-1]; - values = &stack_pointer[-1 - oparg]; - PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); - assert(oparg >= 0 && oparg <= 2); - if (oparg) { - PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]); - if (PyLong_Check(lasti)) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyLong_AsLong(lasti); - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(!_PyErr_Occurred(tstate)); - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD); + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); + int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - Py_DECREF(exc); - goto error; - } - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - assert(exc && PyExceptionInstance_Check(exc)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto exception_unwind; - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); + if (err != 0) CEVAL_GOTO(error); } } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _JUMP_BACKWARD { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); + uint16_t the_counter = read_u16(&this_instr[1].cache); + (void)the_counter; + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + #ifdef _Py_TIER2 + #if ENABLE_SPECIALIZATION + _Py_BackoffCounter counter = this_instr[1].counter; + if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) { + _Py_CODEUNIT *start = this_instr; + /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ + while (oparg > 255) { + oparg >>= 8; + start--; } - assert(STACK_LEVEL() == 0); + _PyExecutorObject *executor; _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; + int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (optimized <= 0) { + this_instr[1].counter = restart_backoff_counter(counter); + if (optimized < 0) CEVAL_GOTO(error); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + this_instr[1].counter = initial_jump_backoff_counter(); + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(tstate->previous_executor == NULL); + tstate->previous_executor = Py_None; + GOTO_TIER_TWO(executor); } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + else { + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); } -#endif - DISPATCH(); + #endif /* ENABLE_SPECIALIZATION */ + #endif /* _Py_TIER2 */ } + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_RESERVED(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_RESERVED(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -27167,125 +6892,75 @@ _TAIL_CALL_RESERVED(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(RESERVED); - assert(0 && "Executing RESERVED instruction."); - Py_FatalError("Executing RESERVED instruction."); + INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + JUMPBY(-oparg); DISPATCH(); + } +} + -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_FORWARD); + JUMPBY(oparg); + DISPATCH(); } +} -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_APPEND); + _PyStackRef list; + _PyStackRef v; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)); + if (err < 0) CEVAL_GOTO(pop_1_error); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LIST_EXTEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LIST_EXTEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -27293,865 +6968,730 @@ _TAIL_CALL_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(RESUME); - PREDICTED(RESUME); - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; - // _LOAD_BYTECODE - { - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) goto error; + INSTRUCTION_STATS(LIST_EXTEND); + _PyStackRef list_st; + _PyStackRef iterable_st; + iterable_st = stack_pointer[-1]; + list_st = stack_pointer[-2 - (oparg-1)]; + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (none_val == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { _PyFrame_SetStackPointer(frame, stack_pointer); - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); - frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; - frame->instr_ptr = bytecode + off; - // Make sure this_instr gets reset correctley for any uops that - // follow - next_instr = frame->instr_ptr; - DISPATCH(); - } - #endif - } - // _MAYBE_INSTRUMENT - { - if (tstate->tracing == 0) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; - } - next_instr = this_instr; - DISPATCH(); - } - } - } - // _QUICKEN_RESUME - { - #if ENABLE_SPECIALIZATION_FT - if (tstate->tracing == 0 && this_instr->op.code == RESUME) { - FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); - } - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _CHECK_PERIODIC_IF_NOT_YIELD_FROM - { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - } } + PyStackRef_CLOSE(iterable_st); + CEVAL_GOTO(pop_1_error); } + assert(Py_IsNone(none_val)); + PyStackRef_CLOSE(iterable_st); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR); + PREDICTED(LOAD_ATTR); + _Py_CODEUNIT* const this_instr = next_instr - 10; + (void)this_instr; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self_or_null = PyStackRef_NULL; + // _SPECIALIZE_LOAD_ATTR + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); } + OPCODE_DEFERRED_INC(LOAD_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + /* Skip 8 cache entries */ + // _LOAD_ATTR { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + PyObject *attr_o; + if (oparg & 1) { + /* Designed to work in tandem with CALL, pushes two values. */ + attr_o = NULL; _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (is_meth) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr_o != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + } + else { + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + meth | NULL | arg1 | ... | argN + */ + PyStackRef_CLOSE(owner); + if (attr_o == NULL) CEVAL_GOTO(pop_1_error); + self_or_null = PyStackRef_NULL; } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + else { + /* Classic, pushes one value. */ + _PyFrame_SetStackPointer(frame, stack_pointer); + attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(owner); + if (attr_o == NULL) CEVAL_GOTO(pop_1_error); + /* We need to define self_or_null on all paths */ + self_or_null = PyStackRef_NULL; } -#endif - DISPATCH(); + attr = PyStackRef_FromPyObjectSteal(attr_o); } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_RESUME_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_ATTR_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_RESUME_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_ATTR_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME_CHECK); - static_assert(0 == 0, "incorrect cache size"); - #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; - #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); - #ifdef Py_GIL_DISABLED - DEOPT_IF(frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME); - #endif + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + assert(type_version != 0); + DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + assert(type_version != 0); + DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _GUARD_TYPE_VERSION { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } + uint32_t type_version = read_u32(&this_instr[4].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + /* Skip 1 cache entry */ + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *getattribute = read_obj(&this_instr[6].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert((oparg & 1) == 0); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + PyTypeObject *cls = Py_TYPE(owner_o); + assert(type_version != 0); + DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)getattribute; + assert(func_version != 0); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 2); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( + tstate, PyStackRef_FromPyObjectNew(f), 2, frame); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). + STACK_SHRINK(1); + new_frame->localsplus[0] = owner; + new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); + frame->return_offset = 10 ; + DISPATCH_INLINED(new_frame); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_RETURN_GENERATOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_RETURN_GENERATOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_GENERATOR); - _PyStackRef res; - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (gen == NULL) goto error; - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = res; - stack_pointer += 1; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _CHECK_MANAGED_OBJECT_HAS_VALUES + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + } + // _LOAD_ATTR_INSTANCE_VALUE + { + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = *value_ptr; + DEOPT_IF(attr_o == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr_o); + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectSteal(attr_o); + PyStackRef_CLOSE(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _CHECK_ATTR_METHOD_LAZY_DICT { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } + uint16_t dictoffset = read_u16(&this_instr[4].cache); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = *(PyObject **)ptr; + /* This object has a __dict__, just not yet created */ + DEOPT_IF(dict != NULL, LOAD_ATTR); + } + /* Skip 1 cache entry */ + // _LOAD_ATTR_METHOD_LAZY_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_VALUE); - _PyStackRef retval; - _PyStackRef res; - retval = stack_pointer[-1]; - #if TIER_ONE - assert(frame != entry_frame); - #endif - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(EMPTY()); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(frame->return_offset); - res = temp; - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = res; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + } + // _LOAD_ATTR_METHOD_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + /* Cached method object */ + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + PyDictKeysObject *mod_keys; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_MODULE_PUSH_KEYS + { + owner = stack_pointer[-1]; + uint32_t dict_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; + assert(dict != NULL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR); + mod_keys = keys; } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _LOAD_ATTR_MODULE_FROM_KEYS { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); + uint16_t index = read_u16(&this_instr[4].cache); + assert(mod_keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries)); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); + // Clear mod_keys from stack in case we need to deopt + DEOPT_IF(attr_o == NULL, LOAD_ATTR); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + DEOPT_IF(true, LOAD_ATTR); } + #else + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } + /* Skip 2 cache entries */ + // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + PyStackRef_CLOSE(owner); + attr = PyStackRef_FromPyObjectNew(descr); + } + stack_pointer[-1] = attr; + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND); - PREDICTED(SEND); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef receiver; - _PyStackRef v; - _PyStackRef retval; - // _SPECIALIZE_SEND + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION { - receiver = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Send(receiver, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(SEND); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } - // _SEND + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { - v = stack_pointer[-1]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - PyObject *retval_o; - assert(frame != entry_frame); - if ((tstate->interp->eval_frame == NULL) && - (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && - ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) - { - PyGenObject *gen = (PyGenObject *)receiver_o; - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert( 2 + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2 + oparg); - assert(gen_frame->previous == NULL); - gen_frame->previous = frame; - DISPATCH_INLINED(gen_frame); - } - if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = PyObject_CallMethodOneArg(receiver_o, - &_Py_ID(send), - PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - if (retval_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyGen_FetchStopIterationValue(&retval_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err == 0) { - assert(retval_o != NULL); - JUMPBY(oparg); - } - else { - PyStackRef_CLOSE(v); - goto pop_1_error; - } - } - PyStackRef_CLOSE(v); - retval = PyStackRef_FromPyObjectSteal(retval_o); - } - stack_pointer[-1] = retval; - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + PyStackRef_CLOSE(owner); + attr = PyStackRef_FromPyObjectNew(descr); } + stack_pointer[-1] = attr; + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_SEND_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_ATTR_PROPERTY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_SEND_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_ATTR_PROPERTY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND_GEN); - static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); - _PyStackRef receiver; - _PyStackRef v; - _PyInterpreterFrame *gen_frame; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, SEND); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + } + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_PROPERTY_FRAME + { + PyObject *fget = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_IS_TYPE(fget, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)fget; + PyCodeObject *code = (PyCodeObject *)f->func_code; + DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR); + DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR); + DEOPT_IF(code->co_argcount != 1, LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); + new_frame->localsplus[0] = owner; } - // _SEND_GEN_FRAME + // _SAVE_RETURN_OFFSET { - v = stack_pointer[-1]; - receiver = stack_pointer[-2]; - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); - STAT_INC(SEND, hit); - gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert( 2 + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2 + oparg); - gen_frame->previous = frame; + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } // _PUSH_FRAME { - new_frame = gen_frame; // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); @@ -28168,121 +7708,204 @@ _TAIL_CALL_SEND_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, LLTRACE_RESUME_FRAME(); } DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _LOAD_ATTR_SLOT { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + char *addr = (char *)owner_o + index; + PyObject *attr_o = *(PyObject **)addr; + DEOPT_IF(attr_o == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectNew(attr_o); + PyStackRef_CLOSE(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _CHECK_ATTR_WITH_HINT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + DEOPT_IF(dict == NULL, LOAD_ATTR); + assert(PyDict_CheckExact((PyObject *)dict)); + } + // _LOAD_ATTR_WITH_HINT + { + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject *attr_o; + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), LOAD_ATTR); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, LOAD_ATTR); + attr_o = ep->me_value; + DEOPT_IF(attr_o == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_BUILD_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_BUILD_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_BUILD_CLASS); + _PyStackRef bc; + PyObject *bc_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) CEVAL_GOTO(error); + if (bc_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); } + bc = PyStackRef_FromPyObjectSteal(bc_o); + stack_pointer[0] = bc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } +} -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_COMMON_CONSTANT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_COMMON_CONSTANT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); + _PyStackRef value; + // Keep in sync with _common_constants in opcode.py + // If we ever have more than two constants, use a lookup table + PyObject *val; + if (oparg == CONSTANT_ASSERTIONERROR) { + val = PyExc_AssertionError; + } + else { + assert(oparg == CONSTANT_NOTIMPLEMENTEDERROR); + val = PyExc_NotImplementedError; + } + value = PyStackRef_FromPyObjectImmortal(val); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_SETUP_ANNOTATIONS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_CONST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_SETUP_ANNOTATIONS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_CONST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -28290,151 +7913,25 @@ _TAIL_CALL_SETUP_ANNOTATIONS(_PyInterpreterFrame *frame, _PyStackRef *stack_poin { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(SETUP_ANNOTATIONS); - PyObject *ann_dict; - if (LOCALS() == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when setting up annotations"); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - /* check if __annotations__ in locals()... */ - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) goto error; - if (ann_dict == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - ann_dict = PyDict_New(); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (ann_dict == NULL) goto error; - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - Py_DECREF(ann_dict); - if (err) goto error; - } - else { - Py_DECREF(ann_dict); - } + INSTRUCTION_STATS(LOAD_CONST); + PREDICTED(LOAD_CONST); + _PyStackRef value; + value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_SET_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_CONST_IMMORTAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_SET_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_CONST_IMMORTAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -28442,135 +7939,60 @@ _TAIL_CALL_SET_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(SET_ADD); - _PyStackRef set; - _PyStackRef v; - v = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), - PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - if (err) goto pop_1_error; - stack_pointer += -1; + INSTRUCTION_STATS(LOAD_CONST_IMMORTAL); + static_assert(0 == 0, "incorrect cache size"); + _PyStackRef value; + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + assert(_Py_IsImmortal(obj)); + value = PyStackRef_FromPyObjectImmortal(obj); + stack_pointer[0] = value; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_DEREF); + _PyStackRef value; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *value_o = PyCell_GetRef(cell); + if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); } + value = PyStackRef_FromPyObjectSteal(value_o); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -28578,140 +8000,87 @@ _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(_PyInterpreterFrame *frame, _PyStackRef *stack { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); - _PyStackRef attr_st; - _PyStackRef func_in; - _PyStackRef func_out; - func_in = stack_pointer[-1]; - attr_st = stack_pointer[-2]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); - PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); - func_out = func_in; - assert(PyFunction_Check(func)); - size_t offset = _Py_FunctionAttributeOffsets[oparg]; - assert(offset != 0); - PyObject **ptr = (PyObject **)(((char *)func) + offset); - assert(*ptr == NULL); - *ptr = attr; - stack_pointer[-2] = func_out; - stack_pointer += -1; + INSTRUCTION_STATS(LOAD_FAST); + _PyStackRef value; + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_FAST_AND_CLEAR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_FAST_AND_CLEAR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); + _PyStackRef value; + value = GETLOCAL(oparg); + // do not use SETLOCAL here, it decrefs the old value + GETLOCAL(oparg) = PyStackRef_NULL; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_FAST_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_FAST_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_CHECK); + _PyStackRef value; + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); } + value = PyStackRef_DUP(value_s); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_SET_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_SET_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -28719,135 +8088,160 @@ _TAIL_CALL_SET_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(SET_UPDATE); - _PyStackRef set; - _PyStackRef iterable; - iterable = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), - PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(iterable); - if (err < 0) goto pop_1_error; - stack_pointer += -1; + INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); + _PyStackRef value1; + _PyStackRef value2; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = PyStackRef_DUP(GETLOCAL(oparg1)); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); + _PyStackRef class_dict_st; + _PyStackRef value; + class_dict_st = stack_pointer[-1]; + PyObject *value_o; + PyObject *name; + PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); + assert(class_dict); + assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + CEVAL_GOTO(error); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); + if (!value_o) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + value_o = PyCell_GetRef(cell); + if (value_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); } + } + PyStackRef_CLOSE(class_dict_st); + value = PyStackRef_FromPyObjectSteal(value_o); + stack_pointer[-1] = value; + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); + _PyStackRef mod_or_class_dict; + _PyStackRef v; + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *v_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(mod_or_class_dict); + if (err < 0) CEVAL_GOTO(pop_1_error); + if (v_o == NULL) { + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + CEVAL_GOTO(error); + } + } + else { + /* Slow-path if globals or builtins is not a dict */ + /* namespace 1: globals */ + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) CEVAL_GOTO(error); + if (v_o == NULL) { + /* namespace 2: builtins */ + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) CEVAL_GOTO(error); + if (v_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); + } + } + } + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[-1] = v; + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -28855,161 +8249,57 @@ _TAIL_CALL_STORE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { frame->instr_ptr = next_instr; next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR); - PREDICTED(STORE_ATTR); + INSTRUCTION_STATS(LOAD_GLOBAL); + PREDICTED(LOAD_GLOBAL); _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; - _PyStackRef owner; - _PyStackRef v; - // _SPECIALIZE_STORE_ATTR + _PyStackRef *res; + _PyStackRef null = PyStackRef_NULL; + // _SPECIALIZE_LOAD_GLOBAL { - owner = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_StoreAttr(owner, next_instr, name); + _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } - OPCODE_DEFERRED_INC(STORE_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 3 cache entries */ - // _STORE_ATTR - { - v = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), - name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(owner); - if (err) goto pop_2_error; - } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); + OPCODE_DEFERRED_INC(LOAD_GLOBAL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + // _LOAD_GLOBAL + { + res = &stack_pointer[0]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(*res)) CEVAL_GOTO(error); + null = PyStackRef_NULL; + } + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_GLOBAL_BUILTIN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_GLOBAL_BUILTIN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -29017,505 +8307,542 @@ _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *st { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; + INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + PyDictKeysObject *builtins_keys; + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION_AND_LOCK + // _GUARD_GLOBALS_VERSION { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(type_version != 0); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); - PyTypeObject *tp = Py_TYPE(owner_o); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR); - } + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(keys)); } - // _GUARD_DORV_NO_DICT + // _GUARD_BUILTINS_VERSION_PUSH_KEYS { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (_PyObject_GetManagedDict(owner_o) || - !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { - UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR); - } + uint16_t version = read_u16(&this_instr[3].cache); + PyDictObject *dict = (PyDictObject *)BUILTINS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + builtins_keys = keys; + assert(DK_IS_UNICODE(builtins_keys)); } - // _STORE_ATTR_INSTANCE_VALUE + // _LOAD_GLOBAL_BUILTINS_FROM_KEYS { - value = stack_pointer[-2]; - uint16_t offset = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - STAT_INC(STORE_ATTR, hit); - assert(_PyObject_GetManagedDict(owner_o) == NULL); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *old_value = *value_ptr; - FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); - if (old_value == NULL) { - PyDictValues *values = _PyObject_InlineValues(owner_o); - Py_ssize_t index = value_ptr - values->values; - _PyDictValues_AddToInsertionOrder(values, index); - } - UNLOCK_OBJECT(owner_o); - Py_XDECREF(old_value); - PyStackRef_CLOSE(owner); + uint16_t index = read_u16(&this_instr[4].cache); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + DEOPT_IF(!increfed, LOAD_GLOBAL); + #else + Py_INCREF(res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + null = PyStackRef_NULL; } - stack_pointer += -2; + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_GLOBAL_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_GLOBAL_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + PyDictKeysObject *globals_keys; + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION_PUSH_KEYS + { + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + globals_keys = keys; + assert(DK_IS_UNICODE(globals_keys)); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + /* Skip 1 cache entry */ + // _LOAD_GLOBAL_MODULE_FROM_KEYS { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } + uint16_t index = read_u16(&this_instr[4].cache); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + DEOPT_IF(!increfed, LOAD_GLOBAL); + #else + Py_INCREF(res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + null = PyStackRef_NULL; + } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_LOCALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_LOCALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_LOCALS); + _PyStackRef locals; + PyObject *l = LOCALS(); + if (l == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); + } + locals = PyStackRef_FromPyObjectNew(l); + stack_pointer[0] = locals; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} + - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_NAME); + _PyStackRef v; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) CEVAL_GOTO(error); + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } +} -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_SMALL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_SMALL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SMALL_INT); + _PyStackRef value; + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectImmortal(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_SPECIAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_SPECIAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_SLOT); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SPECIAL); _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); - } - // _STORE_ATTR_SLOT - { - value = stack_pointer[-2]; - uint16_t index = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); - char *addr = (char *)owner_o + index; - STAT_INC(STORE_ATTR, hit); - PyObject *old_value = *(PyObject **)addr; - FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); - UNLOCK_OBJECT(owner_o); - Py_XDECREF(old_value); - PyStackRef_CLOSE(owner); + _PyStackRef attr; + _PyStackRef self_or_null; + owner = stack_pointer[-1]; + assert(oparg <= SPECIAL_MAX); + PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); + PyObject *name = _Py_SpecialMethods[oparg].name; + PyObject *self_or_null_o; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = _PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + _Py_SpecialMethods[oparg].error, + Py_TYPE(owner_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + CEVAL_GOTO(error); } - stack_pointer += -2; + attr = PyStackRef_FromPyObjectSteal(attr_o); + self_or_null = self_or_null_o == NULL ? + PyStackRef_NULL : PyStackRef_FromPyObjectSteal(self_or_null_o); + stack_pointer[0] = attr; + stack_pointer[1] = self_or_null; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR); + PREDICTED(LOAD_SUPER_ATTR); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + // _SPECIALIZE_LOAD_SUPER_ATTR + { + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + int load_method = oparg & 1; + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); } + OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _LOAD_SUPER_ATTR { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); + self_st = stack_pointer[-1]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + CEVAL_GOTO(pop_3_error); + } } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; + // we make no attempt to optimize here; specializations should + // handle any case whose performance we care about + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(super); + } } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); } -#endif - DISPATCH(); + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + if (super == NULL) CEVAL_GOTO(pop_3_error); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(super); + if (attr_o == NULL) CEVAL_GOTO(error); + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; } + stack_pointer[0] = attr; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr_st; /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); - } - // _STORE_ATTR_WITH_HINT - { - value = stack_pointer[-2]; - uint16_t hint = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, STORE_ATTR); - DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR); - #ifdef Py_GIL_DISABLED - if (dict != _PyObject_GetManagedDict(owner_o)) { - UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); - } - #endif - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (hint >= (size_t)dict->ma_keys->dk_nentries || - !DK_IS_UNICODE(dict->ma_keys)) { - UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); - } - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) { - UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); - } - PyObject *old_value = ep->me_value; - if (old_value == NULL) { - UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); - UNLOCK_OBJECT(dict); - // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, - // when dict only holds the strong reference to value in ep->me_value. - Py_XDECREF(old_value); - STAT_INC(STORE_ATTR, hit); - PyStackRef_CLOSE(owner); - } + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(!(oparg & 1)); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + if (attr == NULL) CEVAL_GOTO(pop_3_error); + attr_st = PyStackRef_FromPyObjectSteal(attr); + stack_pointer[-3] = attr_st; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef self_or_null; + /* Skip 1 cache entry */ + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(oparg & 1); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyTypeObject *cls = (PyTypeObject *)class; + int method_found = 0; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + if (attr_o == NULL) { + PyStackRef_CLOSE(self_st); + CEVAL_GOTO(pop_3_error); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } + if (method_found) { + self_or_null = self_st; // transfer ownership + } else { + PyStackRef_CLOSE(self_st); + self_or_null = PyStackRef_NULL; + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MAKE_CELL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MAKE_CELL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_CELL); + // "initial" is probably NULL but not if it's an arg (or set + // via the f_locals proxy before MAKE_CELL has run). + PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *cell = PyCell_New(initial); + if (cell == NULL) { + CEVAL_GOTO(error); } + SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell)); + DISPATCH(); } +} -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MAKE_FUNCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MAKE_FUNCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_FUNCTION); + _PyStackRef codeobj_st; + _PyStackRef func; + codeobj_st = stack_pointer[-1]; + PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(codeobj_st); + if (func_obj == NULL) CEVAL_GOTO(pop_1_error); + _PyFunction_SetVersion( + func_obj, ((PyCodeObject *)codeobj)->co_version); + func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); + stack_pointer[-1] = func; + DISPATCH(); + } } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_MAP_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_MAP_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -29523,131 +8850,125 @@ _TAIL_CALL_STORE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(STORE_DEREF); - _PyStackRef v; - v = stack_pointer[-1]; - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + INSTRUCTION_STATS(MAP_ADD); + _PyStackRef dict_st; + _PyStackRef key; + _PyStackRef value; + value = stack_pointer[-1]; + key = stack_pointer[-2]; + dict_st = stack_pointer[-3 - (oparg - 1)]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + /* dict[key] = value */ + // Do not DECREF INPUTS because the function steals the references _PyFrame_SetStackPointer(frame, stack_pointer); - PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); + int err = _PyDict_SetItem_Take2( + (PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(key), + PyStackRef_AsPyObjectSteal(value) + ); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; + if (err != 0) CEVAL_GOTO(pop_2_error); + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MATCH_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MATCH_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_CLASS); + _PyStackRef subject; + _PyStackRef type; + _PyStackRef names; + _PyStackRef attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; + // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or + // None on failure. + assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attrs_o = _PyEval_MatchClass(tstate, + PyStackRef_AsPyObjectBorrow(subject), + PyStackRef_AsPyObjectBorrow(type), oparg, + PyStackRef_AsPyObjectBorrow(names)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(subject); + PyStackRef_CLOSE(type); + PyStackRef_CLOSE(names); + if (attrs_o) { + assert(PyTuple_CheckExact(attrs_o)); // Success! + attrs = PyStackRef_FromPyObjectSteal(attrs_o); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } + else { + if (_PyErr_Occurred(tstate)) CEVAL_GOTO(pop_3_error); + // Error! + attrs = PyStackRef_None; // Failure! + } + stack_pointer[-3] = attrs; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MATCH_KEYS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MATCH_KEYS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_KEYS); + _PyStackRef subject; + _PyStackRef keys; + _PyStackRef values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; + // On successful match, PUSH(values). Otherwise, PUSH(None). + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, + PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (values_or_none_o == NULL) CEVAL_GOTO(error); + values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); + stack_pointer[0] = values_or_none; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_MATCH_MAPPING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_MATCH_MAPPING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -29655,128 +8976,55 @@ _TAIL_CALL_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(STORE_FAST); - _PyStackRef value; - value = stack_pointer[-1]; - SETLOCAL(oparg, value); - stack_pointer += -1; + INSTRUCTION_STATS(MATCH_MAPPING); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MATCH_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_MATCH_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_SEQUENCE); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_NOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_NOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -29784,131 +9032,39 @@ _TAIL_CALL_STORE_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_p { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); - _PyStackRef value1; - _PyStackRef value2; - value1 = stack_pointer[-1]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - SETLOCAL(oparg1, value1); - value2 = PyStackRef_DUP(GETLOCAL(oparg2)); - stack_pointer[-1] = value2; + INSTRUCTION_STATS(NOP); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOT_TAKEN); + DISPATCH(); + } } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_FAST_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_POP_EXCEPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_FAST_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_POP_EXCEPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -29916,267 +9072,206 @@ _TAIL_CALL_STORE_FAST_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_ { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_STORE_FAST); - _PyStackRef value2; - _PyStackRef value1; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - SETLOCAL(oparg1, value1); - SETLOCAL(oparg2, value2); - stack_pointer += -2; + INSTRUCTION_STATS(POP_EXCEPT); + _PyStackRef exc_value; + exc_value = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XSETREF(exc_info->exc_value, + PyStackRef_IsNone(exc_value) + ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_GLOBAL); - _PyStackRef v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - if (err) goto pop_1_error; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_FALSE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NONE); + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + { + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; + } + else { + b = PyStackRef_False; + PyStackRef_CLOSE(value); } } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _POP_JUMP_IF_TRUE { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); + cond = b; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + { + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); + else { + b = PyStackRef_False; + PyStackRef_CLOSE(value); } + } + // _POP_JUMP_IF_FALSE + { + cond = b; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_TRUE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } +} -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_POP_TOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_POP_TOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_TOP); + _PyStackRef value; + value = stack_pointer[-1]; + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_PUSH_EXC_INFO(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_PUSH_EXC_INFO(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -30184,621 +9279,487 @@ _TAIL_CALL_STORE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(STORE_NAME); - _PyStackRef v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when storing %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - goto pop_1_error; - } - if (PyDict_CheckExact(ns)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + INSTRUCTION_STATS(PUSH_EXC_INFO); + _PyStackRef exc; + _PyStackRef prev_exc; + _PyStackRef new_exc; + exc = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + prev_exc = PyStackRef_None; } - PyStackRef_CLOSE(v); - if (err) goto pop_1_error; - stack_pointer += -1; + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_PUSH_NULL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_PUSH_NULL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_NULL); + _PyStackRef res; + res = PyStackRef_NULL; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_RAISE_VARARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_RAISE_VARARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(RAISE_VARARGS); + _PyStackRef *args; + args = &stack_pointer[-oparg]; + assert(oparg < 3); + PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; + PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = do_raise(tstate, exc, cause); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + assert(oparg == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(exception_unwind); + } + CEVAL_GOTO(error); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_RERAISE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_RERAISE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; - INSTRUCTION_STATS(STORE_SLICE); - _PyStackRef v; - _PyStackRef container; - _PyStackRef start; - _PyStackRef stop; - // _SPECIALIZE_STORE_SLICE - { - // Placeholder until we implement STORE_SLICE specialization - #if ENABLE_SPECIALIZATION - OPCODE_DEFERRED_INC(STORE_SLICE); - #endif /* ENABLE_SPECIALIZATION */ - } - // _STORE_SLICE - { - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - v = stack_pointer[-4]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); - stack_pointer = _PyFrame_GetStackPointer(frame); - int err; - if (slice == NULL) { - err = 1; + INSTRUCTION_STATS(RERAISE); + _PyStackRef *values; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); + assert(oparg >= 0 && oparg <= 2); + if (oparg) { + PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]); + if (PyLong_Check(lasti)) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyLong_AsLong(lasti); + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!_PyErr_Occurred(tstate)); } else { - stack_pointer += -2; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); stack_pointer = _PyFrame_GetStackPointer(frame); - Py_DECREF(slice); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + Py_DECREF(exc); + CEVAL_GOTO(error); } - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(container); - if (err) goto pop_4_error; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -4; + assert(exc && PyExceptionInstance_Check(exc)); + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(exception_unwind); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_RESERVED(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_RESERVED(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESERVED); + assert(0 && "Executing RESERVED instruction."); + Py_FatalError("Executing RESERVED instruction."); + DISPATCH(); + } +} - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME); + PREDICTED(RESUME); + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (bytecode == NULL) CEVAL_GOTO(error); + _PyFrame_SetStackPointer(frame, stack_pointer); + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + // Make sure this_instr gets reset correctley for any uops that + // follow + next_instr = frame->instr_ptr; + DISPATCH(); } + #endif } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _MAYBE_INSTRUMENT { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); + if (tstate->tracing == 0) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + CEVAL_GOTO(error); + } + next_instr = this_instr; + DISPATCH(); } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); + } + // _QUICKEN_RESUME + { + #if ENABLE_SPECIALIZATION_FT + if (tstate->tracing == 0 && this_instr->op.code == RESUME) { + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); } + } + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_RESUME_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_RESUME_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME_CHECK); + static_assert(0 == 0, "incorrect cache size"); + #if defined(__EMSCRIPTEN__) + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + DEOPT_IF(eval_breaker != version, RESUME); + #ifdef Py_GIL_DISABLED + DEOPT_IF(frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME); + #endif + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_RETURN_GENERATOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_RETURN_GENERATOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR); - PREDICTED(STORE_SUBSCR); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef container; - _PyStackRef sub; - _PyStackRef v; - // _SPECIALIZE_STORE_SUBSCR - { - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_StoreSubscr(container, sub, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(STORE_SUBSCR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _STORE_SUBSCR - { - v = stack_pointer[-3]; - /* container[sub] = v */ - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (err) goto pop_3_error; - } - stack_pointer += -3; + next_instr += 1; + INSTRUCTION_STATS(RETURN_GENERATOR); + _PyStackRef res; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (gen == NULL) CEVAL_GOTO(error); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectSteal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_DICT); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef dict_st; - _PyStackRef sub; - /* Skip 1 cache entry */ - sub = stack_pointer[-1]; - dict_st = stack_pointer[-2]; - value = stack_pointer[-3]; - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); - STAT_INC(STORE_SUBSCR, hit); + next_instr += 1; + INSTRUCTION_STATS(RETURN_VALUE); + _PyStackRef retval; + _PyStackRef res; + retval = stack_pointer[-1]; + #if TIER_ONE + assert(frame != entry_frame); + #endif + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(sub), - PyStackRef_AsPyObjectSteal(value)); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(dict_st); - if (err) goto pop_3_error; - stack_pointer += -3; + LOAD_IP(frame->return_offset); + res = temp; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND); + PREDICTED(SEND); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef receiver; + _PyStackRef v; + _PyStackRef retval; + // _SPECIALIZE_SEND + { + receiver = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Send(receiver, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); } + OPCODE_DEFERRED_INC(SEND); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _SEND { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); + v = stack_pointer[-1]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + PyObject *retval_o; + assert(frame != entry_frame); + if ((tstate->interp->eval_frame == NULL) && + (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && + ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver_o; + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2 + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2 + oparg); + assert(gen_frame->previous == NULL); + gen_frame->previous = frame; + DISPATCH_INLINED(gen_frame); + } + if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; + retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); + stack_pointer = _PyFrame_GetStackPointer(frame); } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + retval_o = PyObject_CallMethodOneArg(receiver_o, + &_Py_ID(send), + PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; + if (retval_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyGen_FetchStopIterationValue(&retval_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err == 0) { + assert(retval_o != NULL); + JUMPBY(oparg); + } + else { + PyStackRef_CLOSE(v); + CEVAL_GOTO(pop_1_error); } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); } -#endif - DISPATCH(); + PyStackRef_CLOSE(v); + retval = PyStackRef_FromPyObjectSteal(retval_o); } + stack_pointer[-1] = retval; + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_SEND_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_STORE_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_SEND_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -30806,154 +9767,112 @@ _TAIL_CALL_STORE_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_ { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef list_st; - _PyStackRef sub_st; + INSTRUCTION_STATS(SEND_GEN); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef receiver; + _PyStackRef v; + _PyInterpreterFrame *gen_frame; + _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - value = stack_pointer[-3]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); - // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR); - // Ensure index < len(list) - if (index >= PyList_GET_SIZE(list)) { - UNLOCK_OBJECT(list); - DEOPT_IF(true, STORE_SUBSCR); - } - STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); - PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); - assert(old_value != NULL); - UNLOCK_OBJECT(list); // unlock before decrefs! - Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - PyStackRef_CLOSE(list_st); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, SEND); } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } + // _SEND_GEN_FRAME + { + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + STAT_INC(SEND, hit); + gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2 + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2 + oparg); + gen_frame->previous = frame; } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _PUSH_FRAME { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } + new_frame = gen_frame; + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SETUP_ANNOTATIONS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SETUP_ANNOTATIONS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SETUP_ANNOTATIONS); + PyObject *ann_dict; + if (LOCALS() == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when setting up annotations"); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); } + /* check if __annotations__ in locals()... */ + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) CEVAL_GOTO(error); + if (ann_dict == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + ann_dict = PyDict_New(); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (ann_dict == NULL) CEVAL_GOTO(error); + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(ann_dict); + if (err) CEVAL_GOTO(error); + } + else { + Py_DECREF(ann_dict); + } + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_SWAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_SET_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_SWAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_SET_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -30961,845 +9880,781 @@ _TAIL_CALL_SWAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(SWAP); - _PyStackRef bottom_in; - _PyStackRef top_in; - _PyStackRef top_out; - _PyStackRef bottom_out; - top_in = stack_pointer[-1]; - bottom_in = stack_pointer[-2 - (oparg-2)]; - bottom_out = bottom_in; - top_out = top_in; - assert(oparg >= 2); - stack_pointer[-2 - (oparg-2)] = top_out; - stack_pointer[-1] = bottom_out; + INSTRUCTION_STATS(SET_ADD); + _PyStackRef set; + _PyStackRef v; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + if (err) CEVAL_GOTO(pop_1_error); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); + _PyStackRef attr_st; + _PyStackRef func_in; + _PyStackRef func_out; + func_in = stack_pointer[-1]; + attr_st = stack_pointer[-2]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); + PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); + func_out = func_in; + assert(PyFunction_Check(func)); + size_t offset = _Py_FunctionAttributeOffsets[oparg]; + assert(offset != 0); + PyObject **ptr = (PyObject **)(((char *)func) + offset); + assert(*ptr == NULL); + *ptr = attr; + stack_pointer[-2] = func_out; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SET_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SET_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_UPDATE); + _PyStackRef set; + _PyStackRef iterable; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(iterable); + if (err < 0) CEVAL_GOTO(pop_1_error); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_TO_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_STORE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_TO_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_STORE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL); - PREDICTED(TO_BOOL); - _Py_CODEUNIT* const this_instr = next_instr - 4; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR); + PREDICTED(STORE_ATTR); + _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; - _PyStackRef value; - _PyStackRef res; - // _SPECIALIZE_TO_BOOL + _PyStackRef owner; + _PyStackRef v; + // _SPECIALIZE_STORE_ATTR { - value = stack_pointer[-1]; + owner = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ToBool(value, next_instr); + _Py_Specialize_StoreAttr(owner, next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } - OPCODE_DEFERRED_INC(TO_BOOL); + OPCODE_DEFERRED_INC(STORE_ATTR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION_FT */ } - /* Skip 2 cache entries */ - // _TO_BOOL + /* Skip 3 cache entries */ + // _STORE_ATTR { + v = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (err < 0) goto pop_1_error; - res = err ? PyStackRef_True : PyStackRef_False; + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(owner); + if (err) CEVAL_GOTO(pop_2_error); } - stack_pointer[-1] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION_AND_LOCK + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + DEOPT_IF(true, STORE_ATTR); } } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _GUARD_DORV_NO_DICT { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + DEOPT_IF(true, STORE_ATTR); } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + // _STORE_ATTR_INSTANCE_VALUE + { + value = stack_pointer[-2]; + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + STAT_INC(STORE_ATTR, hit); + assert(_PyObject_GetManagedDict(owner_o) == NULL); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *old_value = *value_ptr; + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); + if (old_value == NULL) { + PyDictValues *values = _PyObject_InlineValues(owner_o); + Py_ssize_t index = value_ptr - values->values; + _PyDictValues_AddToInsertionOrder(values, index); } + UNLOCK_OBJECT(owner_o); + Py_XDECREF(old_value); + PyStackRef_CLOSE(owner); + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + } + // _STORE_ATTR_SLOT + { + value = stack_pointer[-2]; + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + char *addr = (char *)owner_o + index; + STAT_INC(STORE_ATTR, hit); + PyObject *old_value = *(PyObject **)addr; + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); + Py_XDECREF(old_value); + PyStackRef_CLOSE(owner); + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} + + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + } + // _STORE_ATTR_WITH_HINT + { + value = stack_pointer[-2]; + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + DEOPT_IF(dict == NULL, STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR); + #ifdef Py_GIL_DISABLED + if (dict != _PyObject_GetManagedDict(owner_o)) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } + #endif + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (hint >= (size_t)dict->ma_keys->dk_nentries || + !DK_IS_UNICODE(dict->ma_keys)) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + PyObject *old_value = ep->me_value; + if (old_value == NULL) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); } -#endif - DISPATCH(); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(dict); + // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, + // when dict only holds the strong reference to value in ep->me_value. + Py_XDECREF(old_value); + STAT_INC(STORE_ATTR, hit); + PyStackRef_CLOSE(owner); } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } +} -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_DEREF); + _PyStackRef v; + v = stack_pointer[-1]; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef owner; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST); _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL); - } - // _REPLACE_WITH_TRUE - { - value = owner; - PyStackRef_CLOSE(value); - res = PyStackRef_True; - } - stack_pointer[-1] = res; + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); + _PyStackRef value1; + _PyStackRef value2; + value1 = stack_pointer[-1]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[-1] = value2; + DISPATCH(); + } +} - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_FAST_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_FAST_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_STORE_FAST); + _PyStackRef value2; + _PyStackRef value1; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + SETLOCAL(oparg2, value2); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } +} -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_GLOBAL); + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + if (err) CEVAL_GOTO(pop_1_error); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_TO_BOOL_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_STORE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_TO_BOOL_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_STORE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_BOOL); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); + next_instr += 1; + INSTRUCTION_STATS(STORE_NAME); + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when storing %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + CEVAL_GOTO(pop_1_error); } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } + if (PyDict_CheckExact(ns)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); } + PyStackRef_CLOSE(v); + if (err) CEVAL_GOTO(pop_1_error); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_TO_BOOL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_STORE_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_TO_BOOL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_STORE_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_INT); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL); - STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value_o)) { - assert(_Py_IsImmortal(value_o)); - res = PyStackRef_False; - } - else { - PyStackRef_CLOSE(value); - res = PyStackRef_True; - } - stack_pointer[-1] = res; - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } + next_instr += 1; + INSTRUCTION_STATS(STORE_SLICE); + _PyStackRef v; + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + // _SPECIALIZE_STORE_SLICE + { + // Placeholder until we implement STORE_SLICE specialization + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(STORE_SLICE); + #endif /* ENABLE_SPECIALIZATION */ } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _STORE_SLICE { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); + int err; + if (slice == NULL) { + err = 1; } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); + else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(slice); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); + if (err) CEVAL_GOTO(pop_4_error); + } + stack_pointer += -4; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR); + PREDICTED(STORE_SUBSCR); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef container; + _PyStackRef sub; + _PyStackRef v; + // _SPECIALIZE_STORE_SUBSCR + { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_StoreSubscr(container, sub, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(STORE_SUBSCR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } + // _STORE_SUBSCR + { + v = stack_pointer[-3]; + /* container[sub] = v */ + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (err) CEVAL_GOTO(pop_3_error); + } + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_TO_BOOL_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_STORE_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_TO_BOOL_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_STORE_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_LIST); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); _PyStackRef value; - _PyStackRef res; + _PyStackRef dict_st; + _PyStackRef sub; /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL); - STAT_INC(TO_BOOL, hit); - res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; - PyStackRef_CLOSE(value); - stack_pointer[-1] = res; + sub = stack_pointer[-1]; + dict_st = stack_pointer[-2]; + value = stack_pointer[-3]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + STAT_INC(STORE_SUBSCR, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(sub), + PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(dict_st); + if (err) CEVAL_GOTO(pop_3_error); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } + +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else - assert(_PyErr_Occurred(tstate)); +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_STORE_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef list_st; + _PyStackRef sub_st; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + value = stack_pointer[-3]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + // Ensure nonnegative, zero-or-one-digit ints. + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR); + // Ensure index < len(list) + if (index >= PyList_GET_SIZE(list)) { + UNLOCK_OBJECT(list); + DEOPT_IF(true, STORE_SUBSCR); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); + assert(old_value != NULL); + UNLOCK_OBJECT(list); // unlock before decrefs! + Py_DECREF(old_value); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(list_st); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SWAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_SWAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SWAP); + _PyStackRef bottom_in; + _PyStackRef top_in; + _PyStackRef top_out; + _PyStackRef bottom_out; + top_in = stack_pointer[-1]; + bottom_in = stack_pointer[-2 - (oparg-2)]; + bottom_out = bottom_in; + top_out = top_in; + assert(oparg >= 2); + stack_pointer[-2 - (oparg-2)] = top_out; + stack_pointer[-1] = bottom_out; + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_TO_BOOL_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_TO_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_TO_BOOL_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_TO_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -31807,676 +10662,259 @@ _TAIL_CALL_TO_BOOL_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { frame->instr_ptr = next_instr; next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_NONE); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + INSTRUCTION_STATS(TO_BOOL); + PREDICTED(TO_BOOL); + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; _PyStackRef value; _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - res = PyStackRef_False; - stack_pointer[-1] = res; - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + // _SPECIALIZE_TO_BOOL { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); + value = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + _Py_Specialize_ToBool(value, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); } -#endif - DISPATCH(); + OPCODE_DEFERRED_INC(TO_BOOL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } + /* Skip 2 cache entries */ + // _TO_BOOL + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (err < 0) CEVAL_GOTO(pop_1_error); + res = err ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-1] = res; + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_TO_BOOL_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_TO_BOOL_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_STR); + INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef owner; _PyStackRef value; _PyStackRef res; /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL); - STAT_INC(TO_BOOL, hit); - if (value_o == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value_o)); - res = PyStackRef_False; + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL); } - else { - assert(Py_SIZE(value_o)); + // _REPLACE_WITH_TRUE + { + value = owner; PyStackRef_CLOSE(value); res = PyStackRef_True; } stack_pointer[-1] = res; - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_UNARY_INVERT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_TO_BOOL_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_UNARY_INVERT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_TO_BOOL_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_INVERT); + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_BOOL); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; - _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ value = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (res_o == NULL) goto pop_1_error; - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; + DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_TO_BOOL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_TO_BOOL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_INT); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL); + STAT_INC(TO_BOOL, hit); + if (_PyLong_IsZero((PyLongObject *)value_o)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; } + else { + PyStackRef_CLOSE(value); + res = PyStackRef_True; + } + stack_pointer[-1] = res; + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_UNARY_NEGATIVE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_TO_BOOL_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_UNARY_NEGATIVE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_TO_BOOL_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NEGATIVE); + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_LIST); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ value = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL); + STAT_INC(TO_BOOL, hit); + res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; PyStackRef_CLOSE(value); - if (res_o == NULL) goto pop_1_error; - res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_TO_BOOL_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_TO_BOOL_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_NONE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + // This one is a bit weird, because we expect *some* failures: + DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + res = PyStackRef_False; + stack_pointer[-1] = res; + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_UNARY_NOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_TO_BOOL_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_UNARY_NOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_TO_BOOL_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NOT); + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_STR); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ value = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(value)); - res = PyStackRef_IsFalse(value) - ? PyStackRef_True : PyStackRef_False; - stack_pointer[-1] = res; - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL); + STAT_INC(TO_BOOL, hit); + if (value_o == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); + else { + assert(Py_SIZE(value_o)); + PyStackRef_CLOSE(value); + res = PyStackRef_True; } + stack_pointer[-1] = res; + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_UNPACK_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_UNARY_INVERT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_UNPACK_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_UNARY_INVERT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -32484,441 +10922,174 @@ _TAIL_CALL_UNPACK_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(UNPACK_EX); - _PyStackRef seq; - _PyStackRef *right; - seq = stack_pointer[-1]; - right = &stack_pointer[(oparg & 0xFF)]; - _PyStackRef *top = right + (oparg >> 8); + INSTRUCTION_STATS(UNARY_INVERT); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(seq); - if (res == 0) goto pop_1_error; - stack_pointer += (oparg & 0xFF) + (oparg >> 8); - assert(WITHIN_STACK_BOUNDS()); + PyStackRef_CLOSE(value); + if (res_o == NULL) CEVAL_GOTO(pop_1_error); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-1] = res; DISPATCH(); + } +} -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_UNARY_NEGATIVE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_UNARY_NEGATIVE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NEGATIVE); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) CEVAL_GOTO(pop_1_error); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-1] = res; + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_UNPACK_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_UNARY_NOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_UNPACK_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_UNARY_NOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE); - PREDICTED(UNPACK_SEQUENCE); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef seq; - _PyStackRef *output; - // _SPECIALIZE_UNPACK_SEQUENCE - { - seq = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - (void)seq; - (void)counter; - } - // _UNPACK_SEQUENCE - { - output = &stack_pointer[-1]; - _PyStackRef *top = output + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(seq); - if (res == 0) goto pop_1_error; - } - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + next_instr += 1; + INSTRUCTION_STATS(UNARY_NOT); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + stack_pointer[-1] = res; DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_UNPACK_SEQUENCE_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_UNPACK_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_UNPACK_SEQUENCE_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_UNPACK_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { int opcode = next_instr->op.code; { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + next_instr += 1; + INSTRUCTION_STATS(UNPACK_EX); _PyStackRef seq; - _PyStackRef *values; - /* Skip 1 cache entry */ + _PyStackRef *right; seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE); - if (PyList_GET_SIZE(seq_o) != oparg) { - UNLOCK_OBJECT(seq_o); - DEOPT_IF(true, UNPACK_SEQUENCE); - } - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectNew(items[i]); - } - UNLOCK_OBJECT(seq_o); + right = &stack_pointer[(oparg & 0xFF)]; + _PyStackRef *top = right + (oparg >> 8); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); - stack_pointer += -1 + oparg; + if (res == 0) CEVAL_GOTO(pop_1_error); + stack_pointer += (oparg & 0xFF) + (oparg >> 8); assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + } +} + -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_UNPACK_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_UNPACK_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE); + PREDICTED(UNPACK_SEQUENCE); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef seq; + _PyStackRef *output; + // _SPECIALIZE_UNPACK_SEQUENCE + { + seq = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + (void)seq; + (void)counter; } + // _UNPACK_SEQUENCE + { + output = &stack_pointer[-1]; + _PyStackRef *top = output + oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(seq); + if (res == 0) CEVAL_GOTO(pop_1_error); + } + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_UNPACK_SEQUENCE_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_UNPACK_SEQUENCE_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -32926,7 +11097,7 @@ _TAIL_CALL_UNPACK_SEQUENCE_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_ { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); + INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef seq; _PyStackRef *values; @@ -32934,132 +11105,33 @@ _TAIL_CALL_UNPACK_SEQUENCE_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE); + DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE); + if (PyList_GET_SIZE(seq_o) != oparg) { + UNLOCK_OBJECT(seq_o); + DEOPT_IF(true, UNPACK_SEQUENCE); + } STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq_o); + PyObject **items = _PyList_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { *values++ = PyStackRef_FromPyObjectNew(items[i]); } + UNLOCK_OBJECT(seq_o); PyStackRef_CLOSE(seq); stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } #ifdef LLTRACE __attribute__((preserve_none)) PyObject * -_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else __attribute__((preserve_none)) PyObject * -_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, +_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif { @@ -33067,131 +11139,64 @@ _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *st { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef seq; - _PyStackRef val1; - _PyStackRef val0; + _PyStackRef *values; /* Skip 1 cache entry */ seq = stack_pointer[-1]; - assert(oparg == 2); + values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); - val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); - val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - PyStackRef_CLOSE(seq); - stack_pointer[-1] = val1; - stack_pointer[0] = val0; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } + PyObject **items = _PyTuple_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + PyStackRef_CLOSE(seq); + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; +{ + int opcode = next_instr->op.code; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + /* Skip 1 cache entry */ + seq = stack_pointer[-1]; + assert(oparg == 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + PyStackRef_CLOSE(seq); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } @@ -33248,117 +11253,13 @@ _TAIL_CALL_WITH_EXCEPT_START(_PyInterpreterFrame *frame, _PyStackRef *stack_poin PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) goto error; + if (res_o == NULL) CEVAL_GOTO(error); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } @@ -33419,112 +11320,9 @@ _TAIL_CALL_YIELD_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } + static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [BINARY_OP] = _TAIL_CALL_BINARY_OP, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index d17617cab0266b..30af5beb51e010 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -192,18 +192,21 @@ def error_if( except ValueError: offset = -1 if offset > 0: - self.out.emit(f"goto pop_{offset}_") + self.out.emit(f"CEVAL_GOTO(pop_{offset}_") self.out.emit(label) + self.out.emit(")") self.out.emit(";\n") elif offset == 0: - self.out.emit("goto ") + self.out.emit("CEVAL_GOTO(") self.out.emit(label) + self.out.emit(")") self.out.emit(";\n") else: self.out.emit("{\n") storage.copy().flush(self.out) - self.out.emit("goto ") + self.out.emit("CEVAL_GOTO(") self.out.emit(label) + self.out.emit(")") self.out.emit(";\n") self.out.emit("}\n") return not unconditional @@ -219,7 +222,7 @@ def error_no_pop( next(tkn_iter) # LPAREN next(tkn_iter) # RPAREN next(tkn_iter) # Semi colon - self.out.emit_at("goto error;", tkn) + self.out.emit_at("CEVAL_GOTO(error);", tkn) return False def decref_inputs( diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index 838871763f0c25..504737063f36c3 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -48,7 +48,6 @@ ) from cwriter import CWriter from typing import TextIO -from stack import Stack DEFAULT_OUTPUT = ROOT / "Python/generated_cases_tail_call.c.h" @@ -56,6 +55,33 @@ FOOTER = "#undef TIER_ONE\n" +def function_proto(name: str) -> str: + return f""" +#ifdef LLTRACE +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_{name}(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) +#else +__attribute__((preserve_none)) PyObject * +_TAIL_CALL_{name}(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) +#endif +""" + +def error_handler(out: CWriter, body: str, next: str, emit_tail_call: bool = True): + out.emit("{\n") + out.emit("int opcode = next_instr->op.code;\n") + out.emit(body) + if emit_tail_call: + out.emit("#ifdef LLTRACE\n") + out.emit("__attribute__((musttail))\n") + out.emit(f"return _TAIL_CALL_{next}(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace);\n") + out.emit("#else\n") + out.emit("__attribute__((musttail))\n") + out.emit(f"return _TAIL_CALL_{next}(frame, stack_pointer, tstate, next_instr, oparg, entry_frame);\n") + out.emit("#endif\n") + out.emit("}\n") + def generate_tier1( filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool @@ -71,58 +97,42 @@ def generate_tier1( ) out = CWriter(outfile, 0, lines) out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256];\n"); - emitter = Emitter(out) - out.emit("\n") - for name, inst in sorted(analysis.instructions.items()): - out.emit("\n") - out.emit(f""" -#ifdef LLTRACE -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_{name}(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else -__attribute__((preserve_none)) PyObject * -_TAIL_CALL_{name}(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -""") - out.emit("{\n") - # out.emit(f'printf("{name}\\n");\n') - out.emit("int opcode = next_instr->op.code;\n") - out.emit("{\n") - write_single_inst(out, emitter, name, inst) - if not inst.parts[-1].properties.always_exits: - out.emit("DISPATCH();\n") - out.emit(""" -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: + # Emit error handlers + + out.emit(function_proto("error")) + out.emit(";\n") + + out.emit(function_proto("resume_with_error")) + body = """ + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + """ + next = "error" + error_handler(out, body, next) + + out.emit(function_proto("exit_unwind")) + body = """ + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + """ + next = "resume_with_error" + error_handler(out, body, next) + + out.emit(function_proto("exception_unwind")) + body = """ { /* We can't use frame->instr_ptr here, as RERAISE may have set it */ int offset = INSTR_OFFSET()-1; @@ -139,7 +149,7 @@ def generate_tier1( assert(STACK_LEVEL() == 0); _PyFrame_SetStackPointer(frame, stack_pointer); monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; + CEVAL_GOTO(exit_unwind); } assert(STACK_LEVEL() >= level); @@ -151,7 +161,7 @@ def generate_tier1( int frame_lasti = _PyInterpreterFrame_LASTI(frame); PyObject *lasti = PyLong_FromLong(frame_lasti); if (lasti == NULL) { - goto exception_unwind; + CEVAL_GOTO(exception_unwind); } PUSH(PyStackRef_FromPyObjectSteal(lasti)); } @@ -165,7 +175,7 @@ def generate_tier1( next_instr = _PyFrame_GetBytecode(frame) + handler; if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; + CEVAL_GOTO(exception_unwind); } /* Resume normal execution */ #ifdef LLTRACE @@ -175,32 +185,73 @@ def generate_tier1( #endif DISPATCH(); } - } +""" + next = "no target" + error_handler(out, body, next, emit_tail_call=False) -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } + out.emit(function_proto("error")) + body = """ + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -""") + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); +""" + next = "exception_unwind" + error_handler(out, body, next) + + out.emit(function_proto("pop_1_error")) + body = "STACK_SHRINK(1);\n" + next = "error" + error_handler(out, body, next) + + out.emit(function_proto("pop_2_error")) + body = "STACK_SHRINK(1);\n" + next = "pop_1_error" + error_handler(out, body, next) + + out.emit(function_proto("pop_3_error")) + body = "STACK_SHRINK(1);\n" + next = "pop_2_error" + error_handler(out, body, next) + + out.emit(function_proto("pop_4_error")) + body = "STACK_SHRINK(1);\n" + next = "pop_3_error" + error_handler(out, body, next) + + emitter = Emitter(out) + out.emit("\n") + for name, inst in sorted(analysis.instructions.items()): + out.emit("\n") + out.emit(function_proto(name)) + out.emit("{\n") + # out.emit(f'printf("{name}\\n");\n') + out.emit("int opcode = next_instr->op.code;\n") + out.emit("{\n") + write_single_inst(out, emitter, name, inst) + if not inst.parts[-1].properties.always_exits: + out.emit("DISPATCH();\n") out.start_line() out.emit("}\n") + out.emit("}\n") + + out.emit("\n") + out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256] = {\n") for name in sorted(analysis.instructions.keys()): out.emit(f"[{name}] = _TAIL_CALL_{name},\n") From 4db185330a87fc46c663f6e4d1d5c6b782126b96 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 5 Jan 2025 19:03:02 +0800 Subject: [PATCH 005/110] fix on release builds --- Python/ceval.c | 4 +- Python/ceval_macros.h | 2 +- Python/generated_cases_tail_call.c.h | 900 +++++++++--------- .../tier1_tail_call_generator.py | 4 +- 4 files changed, 455 insertions(+), 455 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 1f7515e22a52dc..621c19d61f9678 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -786,14 +786,14 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) #endif #ifdef LLTRACE -PyObject * +static inline PyObject * _TAIL_CALL_shim(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) { return (INSTRUCTION_TABLE[next_instr->op.code])(frame, stack_pointer, tstate, next_instr, next_instr->op.arg, entry_frame, lltrace); } #else -PyObject * +static inline PyObject * _TAIL_CALL_shim(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index f0d2c59d07508c..55272424fc7890 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -182,7 +182,7 @@ do { \ frame = tstate->current_frame = (NEW_FRAME); \ CALL_STAT_INC(inlined_py_calls); \ if (_Py_EnterRecursivePy(tstate)) {\ - goto exit_unwind;\ + CEVAL_GOTO(exit_unwind);\ } \ next_instr = frame->instr_ptr; \ stack_pointer = _PyFrame_GetStackPointer(frame); \ diff --git a/Python/generated_cases_tail_call.c.h b/Python/generated_cases_tail_call.c.h index 7597c2e9fd6936..89ad93f985d6ad 100644 --- a/Python/generated_cases_tail_call.c.h +++ b/Python/generated_cases_tail_call.c.h @@ -10,22 +10,22 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256]; #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif ; #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_resume_with_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_resume_with_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -44,11 +44,11 @@ _TAIL_CALL_resume_with_error(_PyInterpreterFrame *frame, _PyStackRef *stack_poin } #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_exit_unwind(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_exit_unwind(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -79,11 +79,11 @@ _TAIL_CALL_exit_unwind(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_exception_unwind(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_exception_unwind(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -145,11 +145,11 @@ _TAIL_CALL_exception_unwind(_PyInterpreterFrame *frame, _PyStackRef *stack_point } #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -185,11 +185,11 @@ _TAIL_CALL_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_pop_1_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_pop_1_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -206,11 +206,11 @@ _TAIL_CALL_pop_1_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_pop_2_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_pop_2_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -227,11 +227,11 @@ _TAIL_CALL_pop_2_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_pop_3_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_pop_3_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -248,11 +248,11 @@ _TAIL_CALL_pop_3_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_pop_4_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_pop_4_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -271,11 +271,11 @@ _TAIL_CALL_pop_4_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -333,11 +333,11 @@ _TAIL_CALL_BINARY_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_ADD_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_ADD_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -382,11 +382,11 @@ _TAIL_CALL_BINARY_OP_ADD_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_po #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_ADD_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_ADD_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -430,11 +430,11 @@ _TAIL_CALL_BINARY_OP_ADD_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_poin #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -478,11 +478,11 @@ _TAIL_CALL_BINARY_OP_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef *stack_ #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -552,11 +552,11 @@ _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -601,11 +601,11 @@ _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *sta #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_MULTIPLY_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_MULTIPLY_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -649,11 +649,11 @@ _TAIL_CALL_BINARY_OP_MULTIPLY_INT(_PyInterpreterFrame *frame, _PyStackRef *stack #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -698,11 +698,11 @@ _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *sta #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_SUBTRACT_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_SUBTRACT_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -746,11 +746,11 @@ _TAIL_CALL_BINARY_OP_SUBTRACT_INT(_PyInterpreterFrame *frame, _PyStackRef *stack #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -809,11 +809,11 @@ _TAIL_CALL_BINARY_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -869,11 +869,11 @@ _TAIL_CALL_BINARY_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -917,11 +917,11 @@ _TAIL_CALL_BINARY_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_poi #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR_GETITEM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR_GETITEM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -989,11 +989,11 @@ _TAIL_CALL_BINARY_SUBSCR_GETITEM(_PyInterpreterFrame *frame, _PyStackRef *stack_ #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -1042,11 +1042,11 @@ _TAIL_CALL_BINARY_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR_STR_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR_STR_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -1087,11 +1087,11 @@ _TAIL_CALL_BINARY_SUBSCR_STR_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_ #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -1132,11 +1132,11 @@ _TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(_PyInterpreterFrame *frame, _PyStackRef *stac #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -1165,11 +1165,11 @@ _TAIL_CALL_BUILD_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_MAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_MAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -1218,11 +1218,11 @@ _TAIL_CALL_BUILD_MAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -1275,11 +1275,11 @@ _TAIL_CALL_BUILD_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -1318,11 +1318,11 @@ _TAIL_CALL_BUILD_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_STRING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_STRING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -1366,11 +1366,11 @@ _TAIL_CALL_BUILD_STRING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -1399,11 +1399,11 @@ _TAIL_CALL_BUILD_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CACHE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CACHE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -1421,11 +1421,11 @@ _TAIL_CALL_CACHE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -1592,11 +1592,11 @@ _TAIL_CALL_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -1703,11 +1703,11 @@ _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(_PyInterpreterFrame *frame, _PyStackRef *st #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -1819,11 +1819,11 @@ _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -1931,11 +1931,11 @@ _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *st #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BUILTIN_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BUILTIN_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -2021,11 +2021,11 @@ _TAIL_CALL_CALL_BUILTIN_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_poi #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BUILTIN_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BUILTIN_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -2117,11 +2117,11 @@ _TAIL_CALL_CALL_BUILTIN_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_poin #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -2214,11 +2214,11 @@ _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackR #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BUILTIN_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BUILTIN_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -2295,11 +2295,11 @@ _TAIL_CALL_CALL_BUILTIN_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -2474,11 +2474,11 @@ _TAIL_CALL_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_point #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_INTRINSIC_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_INTRINSIC_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -2505,11 +2505,11 @@ _TAIL_CALL_CALL_INTRINSIC_1(_PyInterpreterFrame *frame, _PyStackRef *stack_point #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_INTRINSIC_2(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_INTRINSIC_2(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -2543,11 +2543,11 @@ _TAIL_CALL_CALL_INTRINSIC_2(_PyInterpreterFrame *frame, _PyStackRef *stack_point #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_ISINSTANCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_ISINSTANCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -2600,11 +2600,11 @@ _TAIL_CALL_CALL_ISINSTANCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -2770,11 +2770,11 @@ _TAIL_CALL_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_KW_BOUND_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_KW_BOUND_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -2888,11 +2888,11 @@ _TAIL_CALL_CALL_KW_BOUND_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_p #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_KW_NON_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_KW_NON_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -2993,11 +2993,11 @@ _TAIL_CALL_CALL_KW_NON_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_KW_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_KW_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -3090,11 +3090,11 @@ _TAIL_CALL_CALL_KW_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -3150,11 +3150,11 @@ _TAIL_CALL_CALL_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -3201,11 +3201,11 @@ _TAIL_CALL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_point #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -3299,11 +3299,11 @@ _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(_PyInterpreterFrame *frame, _PyStackRef * #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -3397,11 +3397,11 @@ _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -3482,11 +3482,11 @@ _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(_PyInterpreterFrame *frame, _PyStackRef #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -3570,11 +3570,11 @@ _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(_PyInterpreterFrame *frame, _PyStackRef *sta #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_NON_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_NON_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -3669,11 +3669,11 @@ _TAIL_CALL_CALL_NON_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_po #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_PY_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_PY_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -3763,11 +3763,11 @@ _TAIL_CALL_CALL_PY_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_poi #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -3854,11 +3854,11 @@ _TAIL_CALL_CALL_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_STR_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_STR_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -3918,11 +3918,11 @@ _TAIL_CALL_CALL_STR_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_TUPLE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_TUPLE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -3982,11 +3982,11 @@ _TAIL_CALL_CALL_TUPLE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_TYPE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_TYPE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -4023,11 +4023,11 @@ _TAIL_CALL_CALL_TYPE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CHECK_EG_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CHECK_EG_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -4083,11 +4083,11 @@ _TAIL_CALL_CHECK_EG_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CHECK_EXC_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CHECK_EXC_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -4124,11 +4124,11 @@ _TAIL_CALL_CHECK_EXC_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CLEANUP_THROW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CLEANUP_THROW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -4179,11 +4179,11 @@ _TAIL_CALL_CLEANUP_THROW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_COMPARE_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_COMPARE_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -4253,11 +4253,11 @@ _TAIL_CALL_COMPARE_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_COMPARE_OP_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_COMPARE_OP_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -4304,11 +4304,11 @@ _TAIL_CALL_COMPARE_OP_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_point #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_COMPARE_OP_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_COMPARE_OP_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -4359,11 +4359,11 @@ _TAIL_CALL_COMPARE_OP_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_COMPARE_OP_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_COMPARE_OP_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -4411,11 +4411,11 @@ _TAIL_CALL_COMPARE_OP_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CONTAINS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CONTAINS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -4470,11 +4470,11 @@ _TAIL_CALL_CONTAINS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CONTAINS_OP_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CONTAINS_OP_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -4511,11 +4511,11 @@ _TAIL_CALL_CONTAINS_OP_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_point #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CONTAINS_OP_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CONTAINS_OP_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -4553,11 +4553,11 @@ _TAIL_CALL_CONTAINS_OP_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CONVERT_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_CONVERT_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -4586,11 +4586,11 @@ _TAIL_CALL_CONVERT_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_COPY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_COPY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -4614,11 +4614,11 @@ _TAIL_CALL_COPY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_COPY_FREE_VARS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_COPY_FREE_VARS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -4645,11 +4645,11 @@ _TAIL_CALL_COPY_FREE_VARS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -4675,11 +4675,11 @@ _TAIL_CALL_DELETE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -4706,11 +4706,11 @@ _TAIL_CALL_DELETE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -4737,11 +4737,11 @@ _TAIL_CALL_DELETE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -4772,11 +4772,11 @@ _TAIL_CALL_DELETE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -4814,11 +4814,11 @@ _TAIL_CALL_DELETE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -4848,11 +4848,11 @@ _TAIL_CALL_DELETE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_DICT_MERGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_DICT_MERGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -4890,11 +4890,11 @@ _TAIL_CALL_DICT_MERGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_DICT_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_DICT_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -4936,11 +4936,11 @@ _TAIL_CALL_DICT_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_END_ASYNC_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_END_ASYNC_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -4980,11 +4980,11 @@ _TAIL_CALL_END_ASYNC_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -5005,11 +5005,11 @@ _TAIL_CALL_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -5036,11 +5036,11 @@ _TAIL_CALL_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_ENTER_EXECUTOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_ENTER_EXECUTOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -5082,11 +5082,11 @@ _TAIL_CALL_ENTER_EXECUTOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_EXIT_INIT_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_EXIT_INIT_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -5115,11 +5115,11 @@ _TAIL_CALL_EXIT_INIT_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_EXTENDED_ARG(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_EXTENDED_ARG(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -5139,11 +5139,11 @@ _TAIL_CALL_EXTENDED_ARG(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_FORMAT_SIMPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_FORMAT_SIMPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -5177,11 +5177,11 @@ _TAIL_CALL_FORMAT_SIMPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_FORMAT_WITH_SPEC(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_FORMAT_WITH_SPEC(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -5212,11 +5212,11 @@ _TAIL_CALL_FORMAT_WITH_SPEC(_PyInterpreterFrame *frame, _PyStackRef *stack_point #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -5289,11 +5289,11 @@ _TAIL_CALL_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_FOR_ITER_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_FOR_ITER_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -5350,11 +5350,11 @@ _TAIL_CALL_FOR_ITER_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_FOR_ITER_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_FOR_ITER_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -5414,11 +5414,11 @@ _TAIL_CALL_FOR_ITER_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_FOR_ITER_RANGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_FOR_ITER_RANGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -5472,11 +5472,11 @@ _TAIL_CALL_FOR_ITER_RANGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_FOR_ITER_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_FOR_ITER_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -5533,11 +5533,11 @@ _TAIL_CALL_FOR_ITER_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_AITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_AITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -5593,11 +5593,11 @@ _TAIL_CALL_GET_AITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_ANEXT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_ANEXT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -5626,11 +5626,11 @@ _TAIL_CALL_GET_ANEXT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_AWAITABLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_AWAITABLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -5656,11 +5656,11 @@ _TAIL_CALL_GET_AWAITABLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -5687,11 +5687,11 @@ _TAIL_CALL_GET_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -5721,11 +5721,11 @@ _TAIL_CALL_GET_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_YIELD_FROM_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_YIELD_FROM_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -5777,11 +5777,11 @@ _TAIL_CALL_GET_YIELD_FROM_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_po #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_IMPORT_FROM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_IMPORT_FROM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -5809,11 +5809,11 @@ _TAIL_CALL_IMPORT_FROM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_IMPORT_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_IMPORT_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -5847,11 +5847,11 @@ _TAIL_CALL_IMPORT_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6026,11 +6026,11 @@ _TAIL_CALL_INSTRUMENTED_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_poin #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6046,11 +6046,11 @@ _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6083,11 +6083,11 @@ _TAIL_CALL_INSTRUMENTED_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_p #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6121,11 +6121,11 @@ _TAIL_CALL_INSTRUMENTED_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_p #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6161,11 +6161,11 @@ _TAIL_CALL_INSTRUMENTED_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_ #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6213,11 +6213,11 @@ _TAIL_CALL_INSTRUMENTED_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_ #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_INSTRUCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_INSTRUCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6245,11 +6245,11 @@ _TAIL_CALL_INSTRUMENTED_INSTRUCTION(_PyInterpreterFrame *frame, _PyStackRef *sta #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6282,11 +6282,11 @@ _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *s #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6304,11 +6304,11 @@ _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *st #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_LINE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_LINE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6354,11 +6354,11 @@ _TAIL_CALL_INSTRUMENTED_LINE(_PyInterpreterFrame *frame, _PyStackRef *stack_poin #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6379,11 +6379,11 @@ _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6401,11 +6401,11 @@ _TAIL_CALL_INSTRUMENTED_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6430,11 +6430,11 @@ _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRe #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6461,11 +6461,11 @@ _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6490,11 +6490,11 @@ _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStac #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6519,11 +6519,11 @@ _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6604,11 +6604,11 @@ _TAIL_CALL_INSTRUMENTED_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_po #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6662,11 +6662,11 @@ _TAIL_CALL_INSTRUMENTED_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *st #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_YIELD_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_YIELD_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6744,11 +6744,11 @@ _TAIL_CALL_INSTRUMENTED_YIELD_VALUE(_PyInterpreterFrame *frame, _PyStackRef *sta #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INTERPRETER_EXIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_INTERPRETER_EXIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6776,11 +6776,11 @@ _TAIL_CALL_INTERPRETER_EXIT(_PyInterpreterFrame *frame, _PyStackRef *stack_point #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_IS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_IS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6808,11 +6808,11 @@ _TAIL_CALL_IS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6879,11 +6879,11 @@ _TAIL_CALL_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6905,11 +6905,11 @@ _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(_PyInterpreterFrame *frame, _PyStackRef *s #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6926,11 +6926,11 @@ _TAIL_CALL_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -6955,11 +6955,11 @@ _TAIL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LIST_EXTEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LIST_EXTEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -7005,11 +7005,11 @@ _TAIL_CALL_LIST_EXTEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -7096,11 +7096,11 @@ _TAIL_CALL_LOAD_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -7144,11 +7144,11 @@ _TAIL_CALL_LOAD_ATTR_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -7198,11 +7198,11 @@ _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(_PyInterpreterFrame *frame, _PyS #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -7247,11 +7247,11 @@ _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(_PyInterpreterFrame *frame, _PyStac #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -7305,11 +7305,11 @@ _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *sta #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -7361,11 +7361,11 @@ _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(_PyInterpreterFrame *frame, _PyStackRef *s #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -7410,11 +7410,11 @@ _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *sta #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -7471,11 +7471,11 @@ _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -7536,11 +7536,11 @@ _TAIL_CALL_LOAD_ATTR_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_point #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -7580,11 +7580,11 @@ _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(_PyInterpreterFrame *frame, _PyStackR #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -7635,11 +7635,11 @@ _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(_PyInterpreterFrame *frame, _PySt #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_PROPERTY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_PROPERTY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -7713,11 +7713,11 @@ _TAIL_CALL_LOAD_ATTR_PROPERTY(_PyInterpreterFrame *frame, _PyStackRef *stack_poi #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -7763,11 +7763,11 @@ _TAIL_CALL_LOAD_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -7828,11 +7828,11 @@ _TAIL_CALL_LOAD_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_po #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_BUILD_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_BUILD_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -7865,11 +7865,11 @@ _TAIL_CALL_LOAD_BUILD_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_point #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_COMMON_CONSTANT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_COMMON_CONSTANT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -7900,11 +7900,11 @@ _TAIL_CALL_LOAD_COMMON_CONSTANT(_PyInterpreterFrame *frame, _PyStackRef *stack_p #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_CONST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_CONST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -7926,11 +7926,11 @@ _TAIL_CALL_LOAD_CONST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_CONST_IMMORTAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_CONST_IMMORTAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -7954,11 +7954,11 @@ _TAIL_CALL_LOAD_CONST_IMMORTAL(_PyInterpreterFrame *frame, _PyStackRef *stack_po #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -7987,11 +7987,11 @@ _TAIL_CALL_LOAD_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -8013,11 +8013,11 @@ _TAIL_CALL_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FAST_AND_CLEAR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FAST_AND_CLEAR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -8040,11 +8040,11 @@ _TAIL_CALL_LOAD_FAST_AND_CLEAR(_PyInterpreterFrame *frame, _PyStackRef *stack_po #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FAST_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FAST_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -8075,11 +8075,11 @@ _TAIL_CALL_LOAD_FAST_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -8105,11 +8105,11 @@ _TAIL_CALL_LOAD_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_po #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -8153,11 +8153,11 @@ _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stac #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -8236,11 +8236,11 @@ _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(_PyInterpreterFrame *frame, _PyStackRef *st #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -8294,11 +8294,11 @@ _TAIL_CALL_LOAD_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_GLOBAL_BUILTIN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_GLOBAL_BUILTIN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -8358,11 +8358,11 @@ _TAIL_CALL_LOAD_GLOBAL_BUILTIN(_PyInterpreterFrame *frame, _PyStackRef *stack_po #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_GLOBAL_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_GLOBAL_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -8414,11 +8414,11 @@ _TAIL_CALL_LOAD_GLOBAL_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_poi #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_LOCALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_LOCALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -8447,11 +8447,11 @@ _TAIL_CALL_LOAD_LOCALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -8477,11 +8477,11 @@ _TAIL_CALL_LOAD_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_SMALL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_SMALL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -8504,11 +8504,11 @@ _TAIL_CALL_LOAD_SMALL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_SPECIAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_SPECIAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -8554,11 +8554,11 @@ _TAIL_CALL_LOAD_SPECIAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -8666,11 +8666,11 @@ _TAIL_CALL_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_SUPER_ATTR_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_SUPER_ATTR_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -8714,11 +8714,11 @@ _TAIL_CALL_LOAD_SUPER_ATTR_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_p #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_SUPER_ATTR_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_SUPER_ATTR_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -8775,11 +8775,11 @@ _TAIL_CALL_LOAD_SUPER_ATTR_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_MAKE_CELL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_MAKE_CELL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -8803,11 +8803,11 @@ _TAIL_CALL_MAKE_CELL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_MAKE_FUNCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_MAKE_FUNCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -8837,11 +8837,11 @@ _TAIL_CALL_MAKE_FUNCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_MAP_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_MAP_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -8877,11 +8877,11 @@ _TAIL_CALL_MAP_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_MATCH_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_MATCH_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -8928,11 +8928,11 @@ _TAIL_CALL_MATCH_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_MATCH_KEYS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_MATCH_KEYS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -8963,11 +8963,11 @@ _TAIL_CALL_MATCH_KEYS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_MATCH_MAPPING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_MATCH_MAPPING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -8991,11 +8991,11 @@ _TAIL_CALL_MATCH_MAPPING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_MATCH_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_MATCH_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9019,11 +9019,11 @@ _TAIL_CALL_MATCH_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_NOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_NOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9039,11 +9039,11 @@ _TAIL_CALL_NOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9059,11 +9059,11 @@ _TAIL_CALL_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_EXCEPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_EXCEPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9089,11 +9089,11 @@ _TAIL_CALL_POP_EXCEPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9119,11 +9119,11 @@ _TAIL_CALL_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_poin #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9165,11 +9165,11 @@ _TAIL_CALL_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_point #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9211,11 +9211,11 @@ _TAIL_CALL_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_p #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9241,11 +9241,11 @@ _TAIL_CALL_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_point #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_TOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_TOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9266,11 +9266,11 @@ _TAIL_CALL_POP_TOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_PUSH_EXC_INFO(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_PUSH_EXC_INFO(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9304,11 +9304,11 @@ _TAIL_CALL_PUSH_EXC_INFO(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_PUSH_NULL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_PUSH_NULL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9329,11 +9329,11 @@ _TAIL_CALL_PUSH_NULL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_RAISE_VARARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_RAISE_VARARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9367,11 +9367,11 @@ _TAIL_CALL_RAISE_VARARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_RERAISE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_RERAISE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9424,11 +9424,11 @@ _TAIL_CALL_RERAISE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_RESERVED(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_RESERVED(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9446,11 +9446,11 @@ _TAIL_CALL_RESERVED(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9529,11 +9529,11 @@ _TAIL_CALL_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_RESUME_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_RESUME_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9562,11 +9562,11 @@ _TAIL_CALL_RESUME_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_RETURN_GENERATOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_RETURN_GENERATOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9608,11 +9608,11 @@ _TAIL_CALL_RETURN_GENERATOR(_PyInterpreterFrame *frame, _PyStackRef *stack_point #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9651,11 +9651,11 @@ _TAIL_CALL_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9754,11 +9754,11 @@ _TAIL_CALL_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_SEND_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_SEND_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9819,11 +9819,11 @@ _TAIL_CALL_SEND_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_SETUP_ANNOTATIONS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_SETUP_ANNOTATIONS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9867,11 +9867,11 @@ _TAIL_CALL_SETUP_ANNOTATIONS(_PyInterpreterFrame *frame, _PyStackRef *stack_poin #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_SET_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_SET_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9899,11 +9899,11 @@ _TAIL_CALL_SET_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9936,11 +9936,11 @@ _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(_PyInterpreterFrame *frame, _PyStackRef *stack #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_SET_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_SET_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -9968,11 +9968,11 @@ _TAIL_CALL_SET_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10026,11 +10026,11 @@ _TAIL_CALL_STORE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10095,11 +10095,11 @@ _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *st #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10143,11 +10143,11 @@ _TAIL_CALL_STORE_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10220,11 +10220,11 @@ _TAIL_CALL_STORE_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_p #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10248,11 +10248,11 @@ _TAIL_CALL_STORE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10273,11 +10273,11 @@ _TAIL_CALL_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10301,11 +10301,11 @@ _TAIL_CALL_STORE_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_p #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_FAST_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_FAST_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10331,11 +10331,11 @@ _TAIL_CALL_STORE_FAST_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_ #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10361,11 +10361,11 @@ _TAIL_CALL_STORE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10408,11 +10408,11 @@ _TAIL_CALL_STORE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10469,11 +10469,11 @@ _TAIL_CALL_STORE_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10527,11 +10527,11 @@ _TAIL_CALL_STORE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10567,11 +10567,11 @@ _TAIL_CALL_STORE_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_poin #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10618,11 +10618,11 @@ _TAIL_CALL_STORE_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_ #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_SWAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_SWAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10649,11 +10649,11 @@ _TAIL_CALL_SWAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10702,11 +10702,11 @@ _TAIL_CALL_TO_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10742,11 +10742,11 @@ _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_po #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10769,11 +10769,11 @@ _TAIL_CALL_TO_BOOL_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10807,11 +10807,11 @@ _TAIL_CALL_TO_BOOL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10839,11 +10839,11 @@ _TAIL_CALL_TO_BOOL_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10870,11 +10870,11 @@ _TAIL_CALL_TO_BOOL_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10909,11 +10909,11 @@ _TAIL_CALL_TO_BOOL_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNARY_INVERT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNARY_INVERT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10939,11 +10939,11 @@ _TAIL_CALL_UNARY_INVERT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNARY_NEGATIVE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNARY_NEGATIVE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10969,11 +10969,11 @@ _TAIL_CALL_UNARY_NEGATIVE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNARY_NOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNARY_NOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -10996,11 +10996,11 @@ _TAIL_CALL_UNARY_NOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNPACK_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNPACK_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -11028,11 +11028,11 @@ _TAIL_CALL_UNPACK_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNPACK_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNPACK_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -11084,11 +11084,11 @@ _TAIL_CALL_UNPACK_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -11126,11 +11126,11 @@ _TAIL_CALL_UNPACK_SEQUENCE_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_p #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -11163,11 +11163,11 @@ _TAIL_CALL_UNPACK_SEQUENCE_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_ #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -11201,11 +11201,11 @@ _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *st #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_WITH_EXCEPT_START(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_WITH_EXCEPT_START(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif @@ -11264,11 +11264,11 @@ _TAIL_CALL_WITH_EXCEPT_START(_PyInterpreterFrame *frame, _PyStackRef *stack_poin #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_YIELD_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_YIELD_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index 504737063f36c3..7c365fdbca8fc2 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -58,11 +58,11 @@ def function_proto(name: str) -> str: return f""" #ifdef LLTRACE -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_{name}(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else -__attribute__((preserve_none)) PyObject * +__attribute__((preserve_none)) static PyObject * _TAIL_CALL_{name}(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) #endif From 3bab1b7d4836260e5ecaf3b577023a20077c09e6 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 5 Jan 2025 19:03:52 +0800 Subject: [PATCH 006/110] Garret gets credit for the idea of splitting exits Co-Authored-By: Garrett Gu --- Tools/cases_generator/tier1_tail_call_generator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index 7c365fdbca8fc2..8aa038acd13e8b 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -62,6 +62,7 @@ def function_proto(name: str) -> str: _TAIL_CALL_{name}(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_{name}(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) From e545c555d59544dcaf755898bec0b8e3f48125ef Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 5 Jan 2025 23:05:49 +0800 Subject: [PATCH 007/110] fix jit build --- Python/ceval_macros.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 55272424fc7890..18572df7d7a152 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -364,7 +364,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { if ((COND)) { \ /* This is only a single jump on release builds! */ \ UPDATE_MISS_STATS((INSTNAME)); \ - assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); \ + /* assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); */ \ GO_TO_INSTRUCTION(INSTNAME); \ } @@ -496,7 +496,7 @@ do { \ tstate->previous_executor = NULL; \ frame = tstate->current_frame; \ if (next_instr == NULL) { \ - goto resume_with_error; \ + CEVAL_GOTO(resume_with_error); \ } \ stack_pointer = _PyFrame_GetStackPointer(frame); \ DISPATCH(); \ From 465467f475534c7d8c12d83d28c3349a96e2f2ec Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 5 Jan 2025 23:34:26 +0800 Subject: [PATCH 008/110] force on --- Include/Python.h | 1 + 1 file changed, 1 insertion(+) diff --git a/Include/Python.h b/Include/Python.h index 64be80145890a3..34f161aef5e07d 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -8,6 +8,7 @@ // Since this is a "meta-include" file, "#ifdef __cplusplus / extern "C" {" // is not needed. +#define Py_TAIL_CALL_INTERP 1 // Include Python header files #include "patchlevel.h" From 52b7ecee3f9e45616a8650184301b9d37ad44b33 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 6 Jan 2025 09:42:09 +0800 Subject: [PATCH 009/110] ignore out entire interp loop --- Python/ceval.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 621c19d61f9678..f02cf5b785c98f 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -923,15 +923,15 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif { +#ifndef Py_TAIL_CALL_INTERP /* Start instructions */ -#if !USE_COMPUTED_GOTOS && !defined(Py_TAIL_CALL_INTERP) +#if !USE_COMPUTED_GOTOS dispatch_opcode: switch (opcode) #endif { -#ifndef Py_TAIL_CALL_INTERP #include "generated_cases.c.h" -#endif + #if USE_COMPUTED_GOTOS _unknown_opcode: @@ -953,7 +953,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int /* This should never be reached. Every opcode should end with DISPATCH() or goto error. */ Py_UNREACHABLE(); - +#endif pop_4_error: STACK_SHRINK(1); pop_3_error: From b9bedb13247bb7ceec6fb20866fd348dd0b3aa18 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 7 Jan 2025 03:50:19 +0800 Subject: [PATCH 010/110] Apply changes from Garret's branch --- configure | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configure b/configure index dc5908f82bcd4e..66da2ac44579d7 100755 --- a/configure +++ b/configure @@ -9630,6 +9630,9 @@ fi case $ac_sys_system in #( Emscripten) : +as_fn_append CFLAGS " -mtail-call" + + if test "x$Py_DEBUG" = xyes then : wasm_debug=yes @@ -9712,6 +9715,8 @@ then : fi +as_fn_append CFLAGS " -mtail-call" + as_fn_append LDFLAGS_NODIST " -z stack-size=16777216 -Wl,--stack-first -Wl,--initial-memory=41943040" ;; #( From f1d31903a005168fc468a2151d4f40a5a57e9129 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 7 Jan 2025 07:37:13 +0800 Subject: [PATCH 011/110] Revert "Apply changes from Garret's branch" This reverts commit b9bedb13247bb7ceec6fb20866fd348dd0b3aa18. --- configure | 5 ----- 1 file changed, 5 deletions(-) diff --git a/configure b/configure index 66da2ac44579d7..dc5908f82bcd4e 100755 --- a/configure +++ b/configure @@ -9630,9 +9630,6 @@ fi case $ac_sys_system in #( Emscripten) : -as_fn_append CFLAGS " -mtail-call" - - if test "x$Py_DEBUG" = xyes then : wasm_debug=yes @@ -9715,8 +9712,6 @@ then : fi -as_fn_append CFLAGS " -mtail-call" - as_fn_append LDFLAGS_NODIST " -z stack-size=16777216 -Wl,--stack-first -Wl,--initial-memory=41943040" ;; #( From 6132746c8c18ed8c81a1357832d40bded485ca97 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 10 Jan 2025 04:41:31 +0800 Subject: [PATCH 012/110] Autoconf detection --- Include/Python.h | 2 - configure | 109 ++++++++++++++++++++++++++++++++++------------- configure.ac | 70 +++++++++++++++++++++++------- pyconfig.h.in | 4 +- 4 files changed, 136 insertions(+), 49 deletions(-) diff --git a/Include/Python.h b/Include/Python.h index 34f161aef5e07d..88248cb4e481e6 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -8,8 +8,6 @@ // Since this is a "meta-include" file, "#ifdef __cplusplus / extern "C" {" // is not needed. -#define Py_TAIL_CALL_INTERP 1 - // Include Python header files #include "patchlevel.h" #include "pyconfig.h" diff --git a/configure b/configure index dc5908f82bcd4e..87252fcdde37ee 100755 --- a/configure +++ b/configure @@ -1084,7 +1084,6 @@ enable_shared with_static_libpython enable_profiling enable_gil -enable_tail_call_interp with_pydebug with_trace_refs enable_pystats @@ -1123,6 +1122,7 @@ with_platlibdir with_wheel_pkg_dir with_readline with_computed_gotos +with_tail_calling_interp with_ensurepip with_openssl with_openssl_rpath @@ -1817,8 +1817,6 @@ Optional Features: no) --disable-gil enable experimental support for running without the GIL (default is no) - --enable-tail-call-interp - enable tail calling interpreter --enable-pystats enable internal statistics gathering (default is no) --enable-experimental-jit[=no|yes|yes-off|interpreter] build the experimental just-in-time compiler @@ -1932,6 +1930,9 @@ Optional Packages: use libedit for backend or disable readline module --with-computed-gotos enable computed gotos in evaluation loop (enabled by default on supported compilers) + --tail-calling-interp enable tail-calling interpreter in evaluation loop + and rest of CPython (enabled by default on supported + compilers) --with-ensurepip[=install|upgrade|no] "install" or "upgrade" using bundled pip (default is upgrade) @@ -8238,32 +8239,6 @@ printf "%s\n" "#define Py_GIL_DISABLED 1" >>confdefs.h ABI_THREAD="t" fi -# Check for --enable-tail-call-interp -# --enable-tail-call-interp -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-tail-call-interp" >&5 -printf %s "checking for --enable-tail-call-interp... " >&6; } -# Check whether --enable-tail-call-interp was given. -if test ${enable_tail_call_interp+y} -then : - enableval=$enable_tail_call_interp; -if test "$enableval" != no -then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; }; - -printf "%s\n" "#define Py_TAIL_CALL_INTERP 1" >>confdefs.h - -else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; }; -fi -else case e in #( - e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } ;; -esac -fi - - # Check for --with-pydebug { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-pydebug" >&5 printf %s "checking for --with-pydebug... " >&6; } @@ -29256,6 +29231,82 @@ printf "%s\n" "#define HAVE_COMPUTED_GOTOS 1" >>confdefs.h esac +# Check for --with-tail-calling-interp +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-tail-calling-interp" >&5 +printf %s "checking for --with-tail-calling-interp... " >&6; } + +# Check whether --with-tail-calling-interp was given. +if test ${with_tail_calling_interp+y} +then : + withval=$with_tail_calling_interp; +if test "$withval" = yes +then + +printf "%s\n" "#define Py_TAIL_CALLING_INTERP 1" >>confdefs.h + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } +fi +if test "$withval" = no +then + +printf "%s\n" "#define Py_TAIL_CALLING_INTERP 0" >>confdefs.h + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +fi + +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no value specified" >&5 +printf "%s\n" "no value specified" >&6; } ;; +esac +fi + + + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether $CC supports efficient proper tail calls" >&5 +printf %s "checking whether $CC supports efficient proper tail calls... " >&6; } +if test ${ac_cv_tail_call+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +#if __has_attribute(musttail) +# if __has_attribute(preserve_none) +// Only allowlist x86_64 and AArch64. Add more architectures as compilers gain support for them. +# if defined(__x86_64__) || defined(__aarch64__) +# define PROPER_TAIL_CALLS 1 +# endif +# endif +#endif + +#ifndef PROPER_TAIL_CALLS +# error "No proper tail calls" +#endif + + +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_tail_call=yes +else case e in #( + e) ac_cv_tail_call=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_tail_call" >&5 +printf "%s\n" "$ac_cv_tail_call" >&6; } +case "$ac_cv_tail_call" in yes*) + +printf "%s\n" "#define Py_TAIL_CALLING_INTERP 1" >>confdefs.h + +esac + case $ac_sys_system in AIX*) diff --git a/configure.ac b/configure.ac index 721847d17bec0b..cef065c07af6d5 100644 --- a/configure.ac +++ b/configure.ac @@ -1717,22 +1717,6 @@ then ABI_THREAD="t" fi -# Check for --enable-tail-call-interp -# --enable-tail-call-interp -AC_MSG_CHECKING([for --enable-tail-call-interp]) -AC_ARG_ENABLE([tail-call-interp], - [AS_HELP_STRING([--enable-tail-call-interp], [enable tail calling interpreter]) ], -[ -if test "$enableval" != no -then - AC_MSG_RESULT([yes]); - AC_DEFINE([Py_TAIL_CALL_INTERP], [1], - [Define if you want to enable tail calling interpreter]) -else - AC_MSG_RESULT([no]); -fi], -[AC_MSG_RESULT([no])]) - # Check for --with-pydebug AC_MSG_CHECKING([for --with-pydebug]) AC_ARG_WITH([pydebug], @@ -7013,6 +6997,60 @@ case "$ac_cv_computed_gotos" in yes*) [Define if the C compiler supports computed gotos.]) esac +# Check for --with-tail-calling-interp +AC_MSG_CHECKING([for --with-tail-calling-interp]) +AC_ARG_WITH( + [tail-calling-interp], + [AS_HELP_STRING( + [--tail-calling-interp], + [enable tail-calling interpreter in evaluation loop and rest of CPython (enabled by default on supported compilers)] + )], +[ +if test "$withval" = yes +then + AC_DEFINE([Py_TAIL_CALLING_INTERP], [1], + [Define if you want to use tail-calling interpreters in CPython.]) + AC_MSG_RESULT([yes]) +fi +if test "$withval" = no +then + AC_DEFINE([Py_TAIL_CALLING_INTERP], [0], + [Define if you want to use tail-calling interpreters in CPython.]) + AC_MSG_RESULT([no]) +fi +], +[AC_MSG_RESULT([no value specified])]) + + +AC_CACHE_CHECK([whether $CC supports efficient proper tail calls], [ac_cv_tail_call], +AC_COMPILE_IFELSE([AC_LANG_SOURCE([[[ + +#if __has_attribute(musttail) +# if __has_attribute(preserve_none) +// Only allowlist x86_64 and AArch64. Add more architectures as compilers gain support for them. +# if defined(__x86_64__) || defined(__aarch64__) +# define PROPER_TAIL_CALLS 1 +# endif +# endif +#endif + +#ifndef PROPER_TAIL_CALLS +# error "No proper tail calls" +#endif + +]]])], +[ac_cv_tail_call=yes], +[ac_cv_tail_call=no], +[if test "${with_tail_calling_interp+set}" = set; then + ac_cv_tail_call="$with_tail_calling_interp -- configured --with(out)-tail-calling-interp" + else + ac_cv_tail_call=no + fi])) +case "$ac_cv_tail_call" in yes*) + AC_DEFINE([Py_TAIL_CALLING_INTERP], [1], + [Define if the C compiler supports efficient proper tail calls.]) +esac + case $ac_sys_system in AIX*) AC_DEFINE([HAVE_BROKEN_PIPE_BUF], [1], diff --git a/pyconfig.h.in b/pyconfig.h.in index 1ee381d98d4fb8..f1629f659795a1 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1718,8 +1718,8 @@ /* The version of SunOS/Solaris as reported by `uname -r' without the dot. */ #undef Py_SUNOS_VERSION -/* Define if you want to enable tail calling interpreter */ -#undef Py_TAIL_CALL_INTERP +/* Define if the C compiler supports efficient proper tail calls. */ +#undef Py_TAIL_CALLING_INTERP /* Define if you want to enable tracing references for debugging purpose */ #undef Py_TRACE_REFS From 16db1282aa906d09679536a660ee9506479606b4 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 10 Jan 2025 05:21:23 +0800 Subject: [PATCH 013/110] cleanup --- Makefile.pre.in | 8 +- Python/ceval.c | 32 +- Python/executor_cases.c.h | 6 +- Python/generated_cases.c.h | 403 +++++++++--------- ...l.c.h => generated_tail_call_handlers.c.h} | 225 ++++++++++ .../tier1_tail_call_generator.py | 44 +- configure | 6 +- configure.ac | 10 +- pyconfig.h.in | 2 +- 9 files changed, 473 insertions(+), 263 deletions(-) rename Python/{generated_cases_tail_call.c.h => generated_tail_call_handlers.c.h} (99%) diff --git a/Makefile.pre.in b/Makefile.pre.in index 67acf0fc520087..0f2f144372d6b1 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1985,7 +1985,7 @@ Objects/mimalloc/page.o: $(srcdir)/Objects/mimalloc/page-queue.c .PHONY: regen-cases regen-cases: \ regen-opcode-ids regen-opcode-targets regen-uop-ids regen-opcode-metadata-py \ - regen-generated-cases regen-executor-cases regen-optimizer-cases \ + regen-generated-cases regen-generated-tail-call-handlers regen-executor-cases regen-optimizer-cases \ regen-opcode-metadata regen-uop-metadata .PHONY: regen-opcode-ids @@ -2018,6 +2018,12 @@ regen-generated-cases: -o $(srcdir)/Python/generated_cases.c.h.new $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new +.PHONY: regen-generated-tail-call-handlers +regen-generated-tail-call-handlers: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier1_tail_call_generator.py \ + -o $(srcdir)/Python/generated_tail_call_handlers.c.h.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Python/generated_tail_call_handlers.c.h $(srcdir)/Python/generated_tail_call_handlers.c.h.new + .PHONY: regen-executor-cases regen-executor-cases: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier2_generator.py \ diff --git a/Python/ceval.c b/Python/ceval.c index f02cf5b785c98f..f1995751ef3cf7 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -782,24 +782,24 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) #endif #ifdef Py_TAIL_CALL_INTERP -#include "generated_cases_tail_call.c.h" +#include "generated_tail_call_handlers.c.h" +# ifdef LLTRACE + static inline PyObject * + _TAIL_CALL_shim(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) + { + return (INSTRUCTION_TABLE[next_instr->op.code])(frame, stack_pointer, tstate, next_instr, next_instr->op.arg, entry_frame, lltrace); + } +# else + static inline PyObject * + _TAIL_CALL_shim(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) + { + return (INSTRUCTION_TABLE[next_instr->op.code])(frame, stack_pointer, tstate, next_instr, next_instr->op.arg, entry_frame); + } +# endif #endif -#ifdef LLTRACE -static inline PyObject * -_TAIL_CALL_shim(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, -PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -{ - return (INSTRUCTION_TABLE[next_instr->op.code])(frame, stack_pointer, tstate, next_instr, next_instr->op.arg, entry_frame, lltrace); -} -#else -static inline PyObject * -_TAIL_CALL_shim(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -{ - return (INSTRUCTION_TABLE[next_instr->op.code])(frame, stack_pointer, tstate, next_instr, next_instr->op.arg, entry_frame); -} -#endif PyObject* _Py_HOT_FUNCTION _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index f7374d52705960..026adafe264092 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1362,7 +1362,7 @@ _PyStackRef res; retval = stack_pointer[-1]; #if TIER_ONE - assert(frame != &entry_frame); + assert(frame != entry_frame); #endif _PyStackRef temp = retval; stack_pointer += -1; @@ -1500,7 +1500,7 @@ // The compiler treats any exception raised here as a failed close() // or throw() call. #if TIER_ONE - assert(frame != &entry_frame); + assert(frame != entry_frame); #endif frame->instr_ptr++; PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); @@ -4907,7 +4907,7 @@ PyObject *res_o = PyLong_FromSsize_t(len_i); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { - GOTO_ERROR(error); + CEVAL_GOTO(error); } PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(arg_stackref); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 98743c27c38524..439f1424f3465e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -49,7 +49,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(lhs); PyStackRef_CLOSE(rhs); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) CEVAL_GOTO(pop_2_error); res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -85,7 +85,7 @@ ((PyFloatObject *)left_o)->ob_fval + ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) CEVAL_GOTO(pop_2_error); res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -120,7 +120,7 @@ PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) CEVAL_GOTO(pop_2_error); res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -155,7 +155,7 @@ PyObject *res_o = PyUnicode_Concat(left_o, right_o); PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) CEVAL_GOTO(pop_2_error); res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -212,7 +212,7 @@ PyUnicode_Append(&temp, right_o); *target_local = PyStackRef_FromPyObjectSteal(temp); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (PyStackRef_IsNull(*target_local)) goto pop_2_error; + if (PyStackRef_IsNull(*target_local)) CEVAL_GOTO(pop_2_error); #if TIER_ONE // The STORE_FAST is already done. This is done here in tier one, // and during trace projection in tier two: @@ -252,7 +252,7 @@ ((PyFloatObject *)left_o)->ob_fval * ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) CEVAL_GOTO(pop_2_error); res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -287,7 +287,7 @@ PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) CEVAL_GOTO(pop_2_error); res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -323,7 +323,7 @@ ((PyFloatObject *)left_o)->ob_fval - ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) CEVAL_GOTO(pop_2_error); res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -358,7 +358,7 @@ PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) CEVAL_GOTO(pop_2_error); res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -408,7 +408,7 @@ assert(WITHIN_STACK_BOUNDS()); } PyStackRef_CLOSE(container); - if (res_o == NULL) goto pop_3_error; + if (res_o == NULL) CEVAL_GOTO(pop_3_error); res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-3] = res; @@ -455,7 +455,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) CEVAL_GOTO(pop_2_error); res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -490,7 +490,7 @@ } PyStackRef_CLOSE(dict_st); PyStackRef_CLOSE(sub_st); - if (rc <= 0) goto pop_2_error; + if (rc <= 0) CEVAL_GOTO(pop_2_error); // not found or error res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -673,7 +673,7 @@ if (list_o == NULL) { stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } list = PyStackRef_FromPyObjectSteal(list_o); stack_pointer[-oparg] = list; @@ -697,7 +697,7 @@ { stack_pointer += -oparg*2; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -713,7 +713,7 @@ if (map_o == NULL) { stack_pointer += -oparg*2; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } map = PyStackRef_FromPyObjectSteal(map_o); stack_pointer[-oparg*2] = map; @@ -739,7 +739,7 @@ { stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } } int err = 0; @@ -756,7 +756,7 @@ { stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } } set = PyStackRef_FromPyObjectSteal(set_o); @@ -787,7 +787,7 @@ if (slice_o == NULL) { stack_pointer += -2 - ((oparg == 3) ? 1 : 0); assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } slice = PyStackRef_FromPyObjectSteal(slice_o); stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; @@ -811,7 +811,7 @@ { stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } } PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); @@ -822,7 +822,7 @@ if (str_o == NULL) { stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } str = PyStackRef_FromPyObjectSteal(str_o); stack_pointer[-oparg] = str; @@ -842,7 +842,7 @@ if (tup_o == NULL) { stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } tup = PyStackRef_FromPyObjectSteal(tup_o); stack_pointer[-oparg] = tup; @@ -938,7 +938,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - goto error; + CEVAL_GOTO(error); } frame->return_offset = 4 ; DISPATCH_INLINED(new_frame); @@ -953,7 +953,7 @@ { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -992,7 +992,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1007,7 +1007,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) CEVAL_GOTO(error); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -1060,7 +1060,7 @@ PyObject *self_o = PyType_GenericAlloc(tp, 0); stack_pointer = _PyFrame_GetStackPointer(frame); if (self_o == NULL) { - goto error; + CEVAL_GOTO(error); } self[0] = PyStackRef_FromPyObjectSteal(self_o); _PyStackRef temp = callable[0]; @@ -1088,7 +1088,7 @@ assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { _PyEval_FrameClearAndPop(tstate, shim); - goto error; + CEVAL_GOTO(error); } init_frame = temp; frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; @@ -1287,7 +1287,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - goto error; + CEVAL_GOTO(error); } new_frame = temp; } @@ -1354,7 +1354,7 @@ { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1369,7 +1369,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1384,7 +1384,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) CEVAL_GOTO(error); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -1433,7 +1433,7 @@ { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1452,7 +1452,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1467,7 +1467,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) CEVAL_GOTO(error); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -1520,7 +1520,7 @@ { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1536,7 +1536,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1551,7 +1551,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) CEVAL_GOTO(error); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -1604,7 +1604,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1619,7 +1619,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) CEVAL_GOTO(error); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -1660,13 +1660,13 @@ int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - goto error; + CEVAL_GOTO(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *tuple_o = PySequence_Tuple(callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (tuple_o == NULL) { - goto error; + CEVAL_GOTO(error); } PyStackRef_CLOSE(callargs); tuple = PyStackRef_FromPyObjectSteal(tuple_o); @@ -1699,7 +1699,7 @@ frame, this_instr, func, arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + CEVAL_GOTO(error); } _PyFrame_SetStackPointer(frame, stack_pointer); result_o = PyObject_Call(func, callargs, kwargs); @@ -1746,7 +1746,7 @@ stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { - goto error; + CEVAL_GOTO(error); } assert( 1 == 1); frame->return_offset = 1; @@ -1770,7 +1770,7 @@ if (result_o == NULL) { stack_pointer += -3 - (oparg & 1); assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } result = PyStackRef_FromPyObjectSteal(result_o); } @@ -1785,7 +1785,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) CEVAL_GOTO(error); stack_pointer += 2 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); } @@ -1808,7 +1808,7 @@ PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (res_o == NULL) goto pop_1_error; + if (res_o == NULL) CEVAL_GOTO(pop_1_error); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; DISPATCH(); @@ -1831,7 +1831,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value2_st); PyStackRef_CLOSE(value1_st); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) CEVAL_GOTO(pop_2_error); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -1870,7 +1870,7 @@ int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); stack_pointer = _PyFrame_GetStackPointer(frame); if (retval < 0) { - goto error; + CEVAL_GOTO(error); } res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -1971,7 +1971,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - goto error; + CEVAL_GOTO(error); } assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4 ; @@ -1989,7 +1989,7 @@ { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } } stack_pointer[-1] = kwnames; @@ -2030,7 +2030,7 @@ if (res_o == NULL) { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2114,7 +2114,7 @@ stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - goto error; + CEVAL_GOTO(error); } new_frame = temp; } @@ -2190,7 +2190,7 @@ { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } } PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); @@ -2211,7 +2211,7 @@ if (res_o == NULL) { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2226,7 +2226,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) CEVAL_GOTO(error); stack_pointer += 2 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -2290,7 +2290,7 @@ stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - goto error; + CEVAL_GOTO(error); } new_frame = temp; } @@ -2352,12 +2352,12 @@ Py_ssize_t len_i = PyObject_Length(arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (len_i < 0) { - goto error; + CEVAL_GOTO(error); } PyObject *res_o = PyLong_FromSsize_t(len_i); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { - GOTO_ERROR(error); + CEVAL_GOTO(error); } PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(arg_stackref); @@ -2394,7 +2394,7 @@ UNLOCK_OBJECT(self_o); PyStackRef_CLOSE(self); PyStackRef_CLOSE(callable); - if (err) goto pop_3_error; + if (err) CEVAL_GOTO(pop_3_error); #if TIER_ONE // Skip the following POP_TOP. This is done here in tier one, and // during trace projection in tier two: @@ -2447,7 +2447,7 @@ { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2465,7 +2465,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2480,7 +2480,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) CEVAL_GOTO(error); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -2532,7 +2532,7 @@ { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2550,7 +2550,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2565,7 +2565,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) CEVAL_GOTO(error); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -2622,7 +2622,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2637,7 +2637,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) CEVAL_GOTO(error); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -2697,7 +2697,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2712,7 +2712,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) CEVAL_GOTO(error); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -2765,7 +2765,7 @@ { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2783,7 +2783,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2798,7 +2798,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) CEVAL_GOTO(error); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -2937,7 +2937,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - goto error; + CEVAL_GOTO(error); } new_frame = temp; } @@ -2994,7 +2994,7 @@ PyObject *res_o = PyObject_Str(arg_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(arg); - if (res_o == NULL) goto pop_3_error; + if (res_o == NULL) CEVAL_GOTO(pop_3_error); res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC @@ -3008,7 +3008,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) CEVAL_GOTO(error); stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); } @@ -3045,7 +3045,7 @@ PyObject *res_o = PySequence_Tuple(arg_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(arg); - if (res_o == NULL) goto pop_3_error; + if (res_o == NULL) CEVAL_GOTO(pop_3_error); res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC @@ -3059,7 +3059,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) CEVAL_GOTO(error); stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); } @@ -3116,7 +3116,7 @@ if (err < 0) { PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); - goto pop_2_error; + CEVAL_GOTO(pop_2_error); } PyObject *match_o = NULL; PyObject *rest_o = NULL; @@ -3126,9 +3126,9 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); - if (res < 0) goto pop_2_error; + if (res < 0) CEVAL_GOTO(pop_2_error); assert((match_o == NULL) == (rest_o == NULL)); - if (match_o == NULL) goto pop_2_error; + if (match_o == NULL) CEVAL_GOTO(pop_2_error); if (!Py_IsNone(match_o)) { stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -3162,7 +3162,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { PyStackRef_CLOSE(right); - goto pop_1_error; + CEVAL_GOTO(pop_1_error); } _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyErr_GivenExceptionMatches(left_o, right_o); @@ -3187,7 +3187,7 @@ last_sent_val_st = stack_pointer[-2]; sub_iter_st = stack_pointer[-3]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - assert(throwflag); + // assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); _PyFrame_SetStackPointer(frame, stack_pointer); int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); @@ -3204,7 +3204,9 @@ _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - goto exception_unwind; + none = PyStackRef_NULL; + value = PyStackRef_NULL; + CEVAL_GOTO(exception_unwind); } stack_pointer[-3] = none; stack_pointer[-2] = value; @@ -3251,7 +3253,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) CEVAL_GOTO(pop_2_error); if (oparg & 16) { stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -3259,7 +3261,7 @@ int res_bool = PyObject_IsTrue(res_o); stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(res_o); - if (res_bool < 0) goto error; + if (res_bool < 0) CEVAL_GOTO(error); res = res_bool ? PyStackRef_True : PyStackRef_False; } else { @@ -3430,7 +3432,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); - if (res < 0) goto pop_2_error; + if (res < 0) CEVAL_GOTO(pop_2_error); b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } stack_pointer[-2] = b; @@ -3459,7 +3461,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); - if (res < 0) goto pop_2_error; + if (res < 0) CEVAL_GOTO(pop_2_error); b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -3488,7 +3490,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); - if (res < 0) goto pop_2_error; + if (res < 0) CEVAL_GOTO(pop_2_error); b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -3510,7 +3512,7 @@ PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (result_o == NULL) goto pop_1_error; + if (result_o == NULL) CEVAL_GOTO(pop_1_error); result = PyStackRef_FromPyObjectSteal(result_o); stack_pointer[-1] = result; DISPATCH(); @@ -3560,7 +3562,7 @@ int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); - if (err) goto pop_1_error; + if (err) CEVAL_GOTO(pop_1_error); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -3578,7 +3580,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + CEVAL_GOTO(error); } Py_DECREF(oldobj); DISPATCH(); @@ -3596,7 +3598,7 @@ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + CEVAL_GOTO(error); } SETLOCAL(oparg, PyStackRef_NULL); DISPATCH(); @@ -3612,14 +3614,14 @@ stack_pointer = _PyFrame_GetStackPointer(frame); // Can't use ERROR_IF here. if (err < 0) { - goto error; + CEVAL_GOTO(error); } if (err == 0) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + CEVAL_GOTO(error); } DISPATCH(); } @@ -3636,7 +3638,7 @@ _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + CEVAL_GOTO(error); } _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_DelItem(ns, name); @@ -3648,7 +3650,7 @@ NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + CEVAL_GOTO(error); } DISPATCH(); } @@ -3668,7 +3670,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); - if (err) goto pop_2_error; + if (err) CEVAL_GOTO(pop_2_error); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -3695,7 +3697,7 @@ _PyEval_FormatKwargsError(tstate, callable_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(update); - goto pop_1_error; + CEVAL_GOTO(pop_1_error); } PyStackRef_CLOSE(update); stack_pointer += -1; @@ -3728,7 +3730,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(update); - goto pop_1_error; + CEVAL_GOTO(pop_1_error); } PyStackRef_CLOSE(update); stack_pointer += -1; @@ -3760,7 +3762,7 @@ _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - goto exception_unwind; + CEVAL_GOTO(exception_unwind); } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -3843,7 +3845,7 @@ "__init__() should return None, not '%.200s'", Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + CEVAL_GOTO(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3876,7 +3878,7 @@ PyObject *res_o = PyObject_Format(value_o, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (res_o == NULL) goto pop_1_error; + if (res_o == NULL) CEVAL_GOTO(pop_1_error); res = PyStackRef_FromPyObjectSteal(res_o); } else { @@ -3900,7 +3902,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); PyStackRef_CLOSE(fmt_spec); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) CEVAL_GOTO(pop_2_error); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -3947,7 +3949,7 @@ int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); stack_pointer = _PyFrame_GetStackPointer(frame); if (!matches) { - goto error; + CEVAL_GOTO(error); } _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_MonitorRaise(tstate, frame, this_instr); @@ -4107,7 +4109,7 @@ r->start = value + r->step; r->len--; PyObject *res = PyLong_FromLong(value); - if (res == NULL) goto error; + if (res == NULL) CEVAL_GOTO(error); next = PyStackRef_FromPyObjectSteal(res); } stack_pointer[0] = next; @@ -4186,13 +4188,13 @@ type->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); - goto pop_1_error; + CEVAL_GOTO(pop_1_error); } _PyFrame_SetStackPointer(frame, stack_pointer); iter_o = (*getter)(obj_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); - if (iter_o == NULL) goto pop_1_error; + if (iter_o == NULL) CEVAL_GOTO(pop_1_error); if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { stack_pointer += -1; @@ -4204,7 +4206,7 @@ Py_TYPE(iter_o)->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(iter_o); - goto error; + CEVAL_GOTO(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; @@ -4222,7 +4224,7 @@ PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); stack_pointer = _PyFrame_GetStackPointer(frame); if (awaitable_o == NULL) { - goto error; + CEVAL_GOTO(error); } awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); stack_pointer[0] = awaitable; @@ -4242,7 +4244,7 @@ PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); - if (iter_o == NULL) goto pop_1_error; + if (iter_o == NULL) CEVAL_GOTO(pop_1_error); iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; DISPATCH(); @@ -4260,7 +4262,7 @@ PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); - if (iter_o == NULL) goto pop_1_error; + if (iter_o == NULL) CEVAL_GOTO(pop_1_error); iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; DISPATCH(); @@ -4277,9 +4279,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) goto error; + if (len_i < 0) CEVAL_GOTO(error); PyObject *len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) goto error; + if (len_o == NULL) CEVAL_GOTO(error); len = PyStackRef_FromPyObjectSteal(len_o); stack_pointer[0] = len; stack_pointer += 1; @@ -4306,7 +4308,7 @@ "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + CEVAL_GOTO(error); } iter = iterable; } @@ -4320,7 +4322,7 @@ PyObject *iter_o = PyObject_GetIter(iterable_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { - goto error; + CEVAL_GOTO(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); PyStackRef_CLOSE(iterable); @@ -4341,7 +4343,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) goto error; + if (res_o == NULL) CEVAL_GOTO(error); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; @@ -4366,7 +4368,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(level); PyStackRef_CLOSE(fromlist); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) CEVAL_GOTO(pop_2_error); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -4428,7 +4430,7 @@ frame, this_instr, function, arg0 ); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; + if (err) CEVAL_GOTO(error); } // _DO_CALL { @@ -4460,7 +4462,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - goto error; + CEVAL_GOTO(error); } frame->return_offset = 4 ; DISPATCH_INLINED(new_frame); @@ -4475,7 +4477,7 @@ { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4514,7 +4516,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4529,7 +4531,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) CEVAL_GOTO(error); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -4566,7 +4568,7 @@ tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; + if (err) CEVAL_GOTO(error); PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); GO_TO_INSTRUCTION(CALL_KW); } @@ -4587,7 +4589,7 @@ int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + CEVAL_GOTO(error); } } PyStackRef_CLOSE(value); @@ -4612,7 +4614,7 @@ int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + CEVAL_GOTO(error); } } val = value; @@ -4643,7 +4645,7 @@ int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); stack_pointer = _PyFrame_GetStackPointer(frame); if (!matches) { - goto error; + CEVAL_GOTO(error); } _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_MonitorRaise(tstate, frame, this_instr); @@ -4671,7 +4673,7 @@ int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_opcode < 0) goto error; + if (next_opcode < 0) CEVAL_GOTO(error); next_instr = this_instr; if (_PyOpcode_Caches[next_opcode]) { PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); @@ -4695,7 +4697,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) CEVAL_GOTO(error); } } // _MONITOR_JUMP_BACKWARD @@ -4734,7 +4736,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (original_opcode < 0) { next_instr = this_instr+1; - goto error; + CEVAL_GOTO(error); } next_instr = frame->instr_ptr; if (next_instr != this_instr) { @@ -4852,7 +4854,7 @@ _Py_CODEUNIT *bytecode = _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) goto error; + if (bytecode == NULL) CEVAL_GOTO(error); _PyFrame_SetStackPointer(frame, stack_pointer); ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4875,7 +4877,7 @@ int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + CEVAL_GOTO(error); } next_instr = this_instr; DISPATCH(); @@ -4891,7 +4893,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) CEVAL_GOTO(error); } } } @@ -4901,7 +4903,7 @@ int err = _Py_call_instrumentation( tstate, oparg > 0, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; + if (err) CEVAL_GOTO(error); if (frame->instr_ptr != this_instr) { /* Instrumentation has jumped */ next_instr = frame->instr_ptr; @@ -4926,13 +4928,13 @@ tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; + if (err) CEVAL_GOTO(error); } // _RETURN_VALUE { retval = val; #if TIER_ONE - assert(frame != &entry_frame); + assert(frame != entry_frame); #endif _PyStackRef temp = retval; stack_pointer += -1; @@ -4972,7 +4974,7 @@ frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + CEVAL_GOTO(error); } if (frame->instr_ptr != this_instr) { next_instr = frame->instr_ptr; @@ -4986,7 +4988,7 @@ // The compiler treats any exception raised here as a failed close() // or throw() call. #if TIER_ONE - assert(frame != &entry_frame); + assert(frame != entry_frame); #endif frame->instr_ptr++; PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); @@ -5030,7 +5032,7 @@ INSTRUCTION_STATS(INTERPRETER_EXIT); _PyStackRef retval; retval = stack_pointer[-1]; - assert(frame == &entry_frame); + assert(frame == entry_frame); assert(_PyFrame_IsIncomplete(frame)); /* Restore previous frame and return. */ tstate->current_frame = frame->previous; @@ -5075,7 +5077,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) CEVAL_GOTO(error); } } // _JUMP_BACKWARD @@ -5100,7 +5102,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (optimized <= 0) { this_instr[1].counter = restart_backoff_counter(counter); - if (optimized < 0) goto error; + if (optimized < 0) CEVAL_GOTO(error); } else { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5151,7 +5153,7 @@ list = stack_pointer[-2 - (oparg-1)]; int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), PyStackRef_AsPyObjectSteal(v)); - if (err < 0) goto pop_1_error; + if (err < 0) CEVAL_GOTO(pop_1_error); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -5185,7 +5187,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(iterable_st); - goto pop_1_error; + CEVAL_GOTO(pop_1_error); } assert(Py_IsNone(none_val)); PyStackRef_CLOSE(iterable_st); @@ -5249,7 +5251,7 @@ meth | NULL | arg1 | ... | argN */ PyStackRef_CLOSE(owner); - if (attr_o == NULL) goto pop_1_error; + if (attr_o == NULL) CEVAL_GOTO(pop_1_error); self_or_null = PyStackRef_NULL; } } @@ -5259,7 +5261,7 @@ attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); - if (attr_o == NULL) goto pop_1_error; + if (attr_o == NULL) CEVAL_GOTO(pop_1_error); /* We need to define self_or_null on all paths */ self_or_null = PyStackRef_NULL; } @@ -5844,13 +5846,13 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) goto error; + if (err < 0) CEVAL_GOTO(error); if (bc_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + CEVAL_GOTO(error); } bc = PyStackRef_FromPyObjectSteal(bc_o); stack_pointer[0] = bc; @@ -5920,7 +5922,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + CEVAL_GOTO(error); } value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; @@ -5969,7 +5971,7 @@ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + CEVAL_GOTO(error); } value = PyStackRef_DUP(value_s); stack_pointer[0] = value; @@ -6012,7 +6014,7 @@ int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - goto error; + CEVAL_GOTO(error); } if (!value_o) { PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); @@ -6021,7 +6023,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + CEVAL_GOTO(error); } } PyStackRef_CLOSE(class_dict_st); @@ -6043,7 +6045,7 @@ int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(mod_or_class_dict); - if (err < 0) goto pop_1_error; + if (err < 0) CEVAL_GOTO(pop_1_error); if (v_o == NULL) { if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) @@ -6064,7 +6066,7 @@ NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); } - goto error; + CEVAL_GOTO(error); } } else { @@ -6075,20 +6077,20 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) goto error; + if (err < 0) CEVAL_GOTO(error); if (v_o == NULL) { /* namespace 2: builtins */ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) goto error; + if (err < 0) CEVAL_GOTO(error); if (v_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + CEVAL_GOTO(error); } } } @@ -6136,7 +6138,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*res)) goto error; + if (PyStackRef_IsNull(*res)) CEVAL_GOTO(error); null = PyStackRef_NULL; } if (oparg & 1) stack_pointer[1] = null; @@ -6250,7 +6252,7 @@ _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + CEVAL_GOTO(error); } locals = PyStackRef_FromPyObjectNew(l); stack_pointer[0] = locals; @@ -6268,7 +6270,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *v_o = _PyEval_LoadName(tstate, frame, name); stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) goto error; + if (v_o == NULL) CEVAL_GOTO(error); v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; @@ -6315,7 +6317,7 @@ Py_TYPE(owner_o)->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); } - goto error; + CEVAL_GOTO(error); } attr = PyStackRef_FromPyObjectSteal(attr_o); self_or_null = self_or_null_o == NULL ? @@ -6375,7 +6377,7 @@ PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); - goto pop_3_error; + CEVAL_GOTO(pop_3_error); } } // we make no attempt to optimize here; specializations should @@ -6407,7 +6409,7 @@ PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); - if (super == NULL) goto pop_3_error; + if (super == NULL) CEVAL_GOTO(pop_3_error); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); @@ -6415,7 +6417,7 @@ PyObject *attr_o = PyObject_GetAttr(super, name); stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(super); - if (attr_o == NULL) goto error; + if (attr_o == NULL) CEVAL_GOTO(error); attr = PyStackRef_FromPyObjectSteal(attr_o); null = PyStackRef_NULL; } @@ -6453,7 +6455,7 @@ PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); - if (attr == NULL) goto pop_3_error; + if (attr == NULL) CEVAL_GOTO(pop_3_error); attr_st = PyStackRef_FromPyObjectSteal(attr); stack_pointer[-3] = attr_st; stack_pointer += -2; @@ -6493,7 +6495,7 @@ PyStackRef_CLOSE(class_st); if (attr_o == NULL) { PyStackRef_CLOSE(self_st); - goto pop_3_error; + CEVAL_GOTO(pop_3_error); } if (method_found) { self_or_null = self_st; // transfer ownership @@ -6518,7 +6520,7 @@ PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *cell = PyCell_New(initial); if (cell == NULL) { - goto error; + CEVAL_GOTO(error); } SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell)); DISPATCH(); @@ -6537,7 +6539,7 @@ PyFunction_New(codeobj, GLOBALS()); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(codeobj_st); - if (func_obj == NULL) goto pop_1_error; + if (func_obj == NULL) CEVAL_GOTO(pop_1_error); _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); @@ -6566,7 +6568,7 @@ PyStackRef_AsPyObjectSteal(value) ); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto pop_2_error; + if (err != 0) CEVAL_GOTO(pop_2_error); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -6600,7 +6602,7 @@ attrs = PyStackRef_FromPyObjectSteal(attrs_o); } else { - if (_PyErr_Occurred(tstate)) goto pop_3_error; + if (_PyErr_Occurred(tstate)) CEVAL_GOTO(pop_3_error); // Error! attrs = PyStackRef_None; // Failure! } @@ -6624,7 +6626,7 @@ PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (values_or_none_o == NULL) goto error; + if (values_or_none_o == NULL) CEVAL_GOTO(error); values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -6862,9 +6864,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - goto exception_unwind; + CEVAL_GOTO(exception_unwind); } - goto error; + CEVAL_GOTO(error); } TARGET(RERAISE) { @@ -6895,7 +6897,7 @@ _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(exc); - goto error; + CEVAL_GOTO(error); } stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -6907,7 +6909,8 @@ _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - goto exception_unwind; + CEVAL_GOTO(exception_unwind); + DISPATCH(); } TARGET(RESERVED) { @@ -6935,7 +6938,7 @@ _Py_CODEUNIT *bytecode = _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) goto error; + if (bytecode == NULL) CEVAL_GOTO(error); _PyFrame_SetStackPointer(frame, stack_pointer); ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6958,7 +6961,7 @@ int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + CEVAL_GOTO(error); } next_instr = this_instr; DISPATCH(); @@ -6982,7 +6985,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) CEVAL_GOTO(error); } } } @@ -7019,7 +7022,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); stack_pointer = _PyFrame_GetStackPointer(frame); - if (gen == NULL) goto error; + if (gen == NULL) CEVAL_GOTO(error); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; @@ -7050,7 +7053,7 @@ _PyStackRef res; retval = stack_pointer[-1]; #if TIER_ONE - assert(frame != &entry_frame); + assert(frame != entry_frame); #endif _PyStackRef temp = retval; stack_pointer += -1; @@ -7104,7 +7107,7 @@ v = stack_pointer[-1]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); PyObject *retval_o; - assert(frame != &entry_frame); + assert(frame != entry_frame); if ((tstate->interp->eval_frame == NULL) && (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) @@ -7152,7 +7155,7 @@ } else { PyStackRef_CLOSE(v); - goto pop_1_error; + CEVAL_GOTO(pop_1_error); } } PyStackRef_CLOSE(v); @@ -7224,24 +7227,24 @@ _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + CEVAL_GOTO(error); } /* check if __annotations__ in locals()... */ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) goto error; + if (err < 0) CEVAL_GOTO(error); if (ann_dict == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); ann_dict = PyDict_New(); stack_pointer = _PyFrame_GetStackPointer(frame); - if (ann_dict == NULL) goto error; + if (ann_dict == NULL) CEVAL_GOTO(error); _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(ann_dict); - if (err) goto error; + if (err) CEVAL_GOTO(error); } else { Py_DECREF(ann_dict); @@ -7262,7 +7265,7 @@ PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); - if (err) goto pop_1_error; + if (err) CEVAL_GOTO(pop_1_error); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -7305,7 +7308,7 @@ PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); - if (err < 0) goto pop_1_error; + if (err < 0) CEVAL_GOTO(pop_1_error); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -7349,7 +7352,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); PyStackRef_CLOSE(owner); - if (err) goto pop_2_error; + if (err) CEVAL_GOTO(pop_2_error); } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -7581,7 +7584,7 @@ int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); - if (err) goto pop_1_error; + if (err) CEVAL_GOTO(pop_1_error); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -7602,7 +7605,7 @@ "no locals found when storing %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); - goto pop_1_error; + CEVAL_GOTO(pop_1_error); } if (PyDict_CheckExact(ns)) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -7615,7 +7618,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(v); - if (err) goto pop_1_error; + if (err) CEVAL_GOTO(pop_1_error); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -7662,7 +7665,7 @@ } PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); - if (err) goto pop_4_error; + if (err) CEVAL_GOTO(pop_4_error); } stack_pointer += -4; assert(WITHIN_STACK_BOUNDS()); @@ -7707,7 +7710,7 @@ PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); - if (err) goto pop_3_error; + if (err) CEVAL_GOTO(pop_3_error); } stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); @@ -7735,7 +7738,7 @@ PyStackRef_AsPyObjectSteal(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(dict_st); - if (err) goto pop_3_error; + if (err) CEVAL_GOTO(pop_3_error); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -7830,7 +7833,7 @@ int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (err < 0) goto pop_1_error; + if (err < 0) CEVAL_GOTO(pop_1_error); res = err ? PyStackRef_True : PyStackRef_False; } stack_pointer[-1] = res; @@ -7977,7 +7980,7 @@ PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (res_o == NULL) goto pop_1_error; + if (res_o == NULL) CEVAL_GOTO(pop_1_error); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; DISPATCH(); @@ -7994,7 +7997,7 @@ PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (res_o == NULL) goto pop_1_error; + if (res_o == NULL) CEVAL_GOTO(pop_1_error); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; DISPATCH(); @@ -8027,7 +8030,7 @@ int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); - if (res == 0) goto pop_1_error; + if (res == 0) CEVAL_GOTO(pop_1_error); stack_pointer += (oparg & 0xFF) + (oparg >> 8); assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -8069,7 +8072,7 @@ int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); - if (res == 0) goto pop_1_error; + if (res == 0) CEVAL_GOTO(pop_1_error); } stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -8196,7 +8199,7 @@ PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) goto error; + if (res_o == NULL) CEVAL_GOTO(error); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; @@ -8215,7 +8218,7 @@ // The compiler treats any exception raised here as a failed close() // or throw() call. #if TIER_ONE - assert(frame != &entry_frame); + assert(frame != entry_frame); #endif frame->instr_ptr++; PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); diff --git a/Python/generated_cases_tail_call.c.h b/Python/generated_tail_call_handlers.c.h similarity index 99% rename from Python/generated_cases_tail_call.c.h rename to Python/generated_tail_call_handlers.c.h index 89ad93f985d6ad..46448980b3c238 100644 --- a/Python/generated_cases_tail_call.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -14,6 +14,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -25,6 +26,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_resume_with_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_resume_with_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -48,6 +50,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_exit_unwind(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_exit_unwind(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -83,6 +86,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_exception_unwind(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_exception_unwind(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -149,6 +153,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -189,6 +194,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_pop_1_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_pop_1_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -210,6 +216,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_pop_2_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_pop_2_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -231,6 +238,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_pop_3_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_pop_3_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -252,6 +260,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_pop_4_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_pop_4_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -275,6 +284,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -337,6 +347,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_ADD_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_ADD_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -386,6 +397,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_ADD_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_ADD_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -434,6 +446,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -482,6 +495,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -556,6 +570,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -605,6 +620,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_MULTIPLY_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_MULTIPLY_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -653,6 +669,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -702,6 +719,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_SUBTRACT_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_OP_SUBTRACT_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -750,6 +768,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -813,6 +832,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -873,6 +893,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -921,6 +942,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR_GETITEM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR_GETITEM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -993,6 +1015,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -1046,6 +1069,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR_STR_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR_STR_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -1091,6 +1115,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -1136,6 +1161,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -1169,6 +1195,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_MAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_MAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -1222,6 +1249,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -1279,6 +1307,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -1322,6 +1351,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_STRING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_STRING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -1370,6 +1400,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_BUILD_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -1403,6 +1434,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CACHE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CACHE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -1425,6 +1457,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -1596,6 +1629,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -1707,6 +1741,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -1823,6 +1858,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -1935,6 +1971,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BUILTIN_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BUILTIN_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -2025,6 +2062,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BUILTIN_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BUILTIN_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -2121,6 +2159,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -2218,6 +2257,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BUILTIN_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_BUILTIN_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -2299,6 +2339,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -2478,6 +2519,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_INTRINSIC_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_INTRINSIC_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -2509,6 +2551,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_INTRINSIC_2(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_INTRINSIC_2(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -2547,6 +2590,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_ISINSTANCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_ISINSTANCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -2604,6 +2648,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -2774,6 +2819,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_KW_BOUND_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_KW_BOUND_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -2892,6 +2938,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_KW_NON_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_KW_NON_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -2997,6 +3044,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_KW_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_KW_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -3094,6 +3142,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -3154,6 +3203,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -3205,6 +3255,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -3303,6 +3354,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -3401,6 +3453,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -3486,6 +3539,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -3574,6 +3628,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_NON_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_NON_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -3673,6 +3728,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_PY_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_PY_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -3767,6 +3823,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -3858,6 +3915,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_STR_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_STR_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -3922,6 +3980,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_TUPLE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_TUPLE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -3986,6 +4045,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_TYPE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CALL_TYPE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -4027,6 +4087,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CHECK_EG_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CHECK_EG_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -4087,6 +4148,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CHECK_EXC_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CHECK_EXC_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -4128,6 +4190,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CLEANUP_THROW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CLEANUP_THROW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -4183,6 +4246,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_COMPARE_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_COMPARE_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -4257,6 +4321,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_COMPARE_OP_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_COMPARE_OP_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -4308,6 +4373,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_COMPARE_OP_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_COMPARE_OP_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -4363,6 +4429,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_COMPARE_OP_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_COMPARE_OP_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -4415,6 +4482,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CONTAINS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CONTAINS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -4474,6 +4542,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CONTAINS_OP_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CONTAINS_OP_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -4515,6 +4584,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CONTAINS_OP_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CONTAINS_OP_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -4557,6 +4627,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CONVERT_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_CONVERT_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -4590,6 +4661,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_COPY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_COPY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -4618,6 +4690,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_COPY_FREE_VARS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_COPY_FREE_VARS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -4649,6 +4722,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -4679,6 +4753,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -4710,6 +4785,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -4741,6 +4817,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -4776,6 +4853,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -4818,6 +4896,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_DELETE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -4852,6 +4931,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_DICT_MERGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_DICT_MERGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -4894,6 +4974,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_DICT_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_DICT_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -4940,6 +5021,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_END_ASYNC_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_END_ASYNC_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -4984,6 +5066,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -5009,6 +5092,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -5040,6 +5124,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_ENTER_EXECUTOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_ENTER_EXECUTOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -5086,6 +5171,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_EXIT_INIT_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_EXIT_INIT_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -5119,6 +5205,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_EXTENDED_ARG(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_EXTENDED_ARG(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -5143,6 +5230,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_FORMAT_SIMPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_FORMAT_SIMPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -5181,6 +5269,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_FORMAT_WITH_SPEC(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_FORMAT_WITH_SPEC(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -5216,6 +5305,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -5293,6 +5383,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_FOR_ITER_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_FOR_ITER_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -5354,6 +5445,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_FOR_ITER_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_FOR_ITER_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -5418,6 +5510,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_FOR_ITER_RANGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_FOR_ITER_RANGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -5476,6 +5569,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_FOR_ITER_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_FOR_ITER_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -5537,6 +5631,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_AITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_AITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -5597,6 +5692,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_ANEXT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_ANEXT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -5630,6 +5726,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_AWAITABLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_AWAITABLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -5660,6 +5757,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -5691,6 +5789,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -5725,6 +5824,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_YIELD_FROM_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_GET_YIELD_FROM_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -5781,6 +5881,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_IMPORT_FROM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_IMPORT_FROM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -5813,6 +5914,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_IMPORT_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_IMPORT_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -5851,6 +5953,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6030,6 +6133,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6050,6 +6154,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6087,6 +6192,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6125,6 +6231,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6165,6 +6272,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6217,6 +6325,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_INSTRUCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_INSTRUCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6249,6 +6358,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6286,6 +6396,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6308,6 +6419,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_LINE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_LINE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6358,6 +6470,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6383,6 +6496,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6405,6 +6519,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6434,6 +6549,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6465,6 +6581,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6494,6 +6611,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6523,6 +6641,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6608,6 +6727,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6666,6 +6786,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_YIELD_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INSTRUMENTED_YIELD_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6748,6 +6869,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INTERPRETER_EXIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_INTERPRETER_EXIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6780,6 +6902,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_IS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_IS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6812,6 +6935,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6883,6 +7007,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6909,6 +7034,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6930,6 +7056,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -6959,6 +7086,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LIST_EXTEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LIST_EXTEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -7009,6 +7137,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -7100,6 +7229,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -7148,6 +7278,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -7202,6 +7333,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -7251,6 +7383,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -7309,6 +7442,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -7365,6 +7499,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -7414,6 +7549,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -7475,6 +7611,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -7540,6 +7677,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -7584,6 +7722,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -7639,6 +7778,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_PROPERTY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_PROPERTY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -7717,6 +7857,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -7767,6 +7908,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -7832,6 +7974,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_BUILD_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_BUILD_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -7869,6 +8012,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_COMMON_CONSTANT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_COMMON_CONSTANT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -7904,6 +8048,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_CONST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_CONST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -7930,6 +8075,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_CONST_IMMORTAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_CONST_IMMORTAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -7958,6 +8104,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -7991,6 +8138,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -8017,6 +8165,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FAST_AND_CLEAR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FAST_AND_CLEAR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -8044,6 +8193,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FAST_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FAST_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -8079,6 +8229,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -8109,6 +8260,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -8157,6 +8309,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -8240,6 +8393,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -8298,6 +8452,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_GLOBAL_BUILTIN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_GLOBAL_BUILTIN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -8362,6 +8517,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_GLOBAL_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_GLOBAL_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -8418,6 +8574,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_LOCALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_LOCALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -8451,6 +8608,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -8481,6 +8639,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_SMALL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_SMALL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -8508,6 +8667,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_SPECIAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_SPECIAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -8558,6 +8718,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -8670,6 +8831,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_SUPER_ATTR_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_SUPER_ATTR_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -8718,6 +8880,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_SUPER_ATTR_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_LOAD_SUPER_ATTR_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -8779,6 +8942,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_MAKE_CELL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_MAKE_CELL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -8807,6 +8971,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_MAKE_FUNCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_MAKE_FUNCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -8841,6 +9006,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_MAP_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_MAP_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -8881,6 +9047,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_MATCH_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_MATCH_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -8932,6 +9099,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_MATCH_KEYS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_MATCH_KEYS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -8967,6 +9135,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_MATCH_MAPPING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_MATCH_MAPPING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -8995,6 +9164,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_MATCH_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_MATCH_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9023,6 +9193,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_NOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_NOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9043,6 +9214,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9063,6 +9235,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_EXCEPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_EXCEPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9093,6 +9266,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9123,6 +9297,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9169,6 +9344,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9215,6 +9391,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9245,6 +9422,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_TOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_POP_TOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9270,6 +9448,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_PUSH_EXC_INFO(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_PUSH_EXC_INFO(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9308,6 +9487,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_PUSH_NULL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_PUSH_NULL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9333,6 +9513,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_RAISE_VARARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_RAISE_VARARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9371,6 +9552,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_RERAISE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_RERAISE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9428,6 +9610,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_RESERVED(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_RESERVED(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9450,6 +9633,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9533,6 +9717,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_RESUME_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_RESUME_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9566,6 +9751,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_RETURN_GENERATOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_RETURN_GENERATOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9612,6 +9798,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9655,6 +9842,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9758,6 +9946,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_SEND_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_SEND_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9823,6 +10012,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_SETUP_ANNOTATIONS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_SETUP_ANNOTATIONS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9871,6 +10061,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_SET_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_SET_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9903,6 +10094,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9940,6 +10132,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_SET_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_SET_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -9972,6 +10165,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10030,6 +10224,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10099,6 +10294,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10147,6 +10343,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10224,6 +10421,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10252,6 +10450,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10277,6 +10476,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10305,6 +10505,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_FAST_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_FAST_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10335,6 +10536,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10365,6 +10567,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10412,6 +10615,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10473,6 +10677,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10531,6 +10736,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10571,6 +10777,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_STORE_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10622,6 +10829,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_SWAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_SWAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10653,6 +10861,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10706,6 +10915,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10746,6 +10956,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10773,6 +10984,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10811,6 +11023,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10843,6 +11056,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10874,6 +11088,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_TO_BOOL_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10913,6 +11128,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNARY_INVERT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNARY_INVERT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10943,6 +11159,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNARY_NEGATIVE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNARY_NEGATIVE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -10973,6 +11190,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNARY_NOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNARY_NOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -11000,6 +11218,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNPACK_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNPACK_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -11032,6 +11251,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNPACK_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNPACK_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -11088,6 +11308,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -11130,6 +11351,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -11167,6 +11389,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -11205,6 +11428,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_WITH_EXCEPT_START(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_WITH_EXCEPT_START(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) @@ -11268,6 +11492,7 @@ __attribute__((preserve_none)) static PyObject * _TAIL_CALL_YIELD_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) #else + __attribute__((preserve_none)) static PyObject * _TAIL_CALL_YIELD_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index 8aa038acd13e8b..68164c3cb62d63 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -1,6 +1,6 @@ -"""Generate the main interpreter tail call. +"""Generate the main interpreter tail call handlers. Reads the instruction definitions from bytecodes.c. -Writes the cases to generated_cases.c.h, which is #included in ceval.c. +Writes the cases to generated_tail_call_handlers.c.h, which is #included in ceval.c. """ import argparse @@ -11,46 +11,21 @@ from generators_common import ( ROOT, write_header, + CWriter, + Emitter, ) -from tier1_generator import ( - write_single_inst -) - -DEFAULT_INPUT = ROOT / "Python/generated_cases.c.h" -DEFAULT_OUTPUT = ROOT / "Python/generated_cases_tail_call.c.h" - - -"""Generate the main interpreter tail calls. -Reads the instruction definitions from bytecodes.c. -Writes the cases to generated_cases.c.h, which is #included in ceval.c. -""" - -import argparse - from analyzer import ( Analysis, - Instruction, - Uop, - Part, analyze_files, - Skip, - Flush, - analysis_error, - StackItem, -) -from generators_common import ( - DEFAULT_INPUT, - ROOT, - write_header, - type_and_null, - Emitter, ) -from cwriter import CWriter -from typing import TextIO +from tier1_generator import ( + write_single_inst +) -DEFAULT_OUTPUT = ROOT / "Python/generated_cases_tail_call.c.h" +DEFAULT_INPUT = ROOT / "Python/generated_tail_call_handlers.c.h" +DEFAULT_OUTPUT = ROOT / "Python/generated_tail_call_handlers.c.h" FOOTER = "#undef TIER_ONE\n" @@ -235,6 +210,7 @@ def generate_tier1( next = "pop_3_error" error_handler(out, body, next) + emitter = Emitter(out) out.emit("\n") for name, inst in sorted(analysis.instructions.items()): diff --git a/configure b/configure index 87252fcdde37ee..48ed56b860ae92 100755 --- a/configure +++ b/configure @@ -29242,7 +29242,7 @@ then : if test "$withval" = yes then -printf "%s\n" "#define Py_TAIL_CALLING_INTERP 1" >>confdefs.h +printf "%s\n" "#define Py_TAIL_CALL_INTERP 1" >>confdefs.h { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } @@ -29250,7 +29250,7 @@ fi if test "$withval" = no then -printf "%s\n" "#define Py_TAIL_CALLING_INTERP 0" >>confdefs.h +printf "%s\n" "#define Py_TAIL_CALL_INTERP 0" >>confdefs.h { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } @@ -29303,7 +29303,7 @@ fi printf "%s\n" "$ac_cv_tail_call" >&6; } case "$ac_cv_tail_call" in yes*) -printf "%s\n" "#define Py_TAIL_CALLING_INTERP 1" >>confdefs.h +printf "%s\n" "#define Py_TAIL_CALL_INTERP 1" >>confdefs.h esac diff --git a/configure.ac b/configure.ac index cef065c07af6d5..f2d4912322b28b 100644 --- a/configure.ac +++ b/configure.ac @@ -7008,13 +7008,13 @@ AC_ARG_WITH( [ if test "$withval" = yes then - AC_DEFINE([Py_TAIL_CALLING_INTERP], [1], + AC_DEFINE([Py_TAIL_CALL_INTERP], [1], [Define if you want to use tail-calling interpreters in CPython.]) AC_MSG_RESULT([yes]) fi if test "$withval" = no then - AC_DEFINE([Py_TAIL_CALLING_INTERP], [0], + AC_DEFINE([Py_TAIL_CALL_INTERP], [0], [Define if you want to use tail-calling interpreters in CPython.]) AC_MSG_RESULT([no]) fi @@ -7041,13 +7041,13 @@ AC_COMPILE_IFELSE([AC_LANG_SOURCE([[[ ]]])], [ac_cv_tail_call=yes], [ac_cv_tail_call=no], -[if test "${with_tail_calling_interp+set}" = set; then - ac_cv_tail_call="$with_tail_calling_interp -- configured --with(out)-tail-calling-interp" +[if test "${with_tail_call_interp+set}" = set; then + ac_cv_tail_call="$with_tail_call_interp -- configured --with(out)-tail-call-interp" else ac_cv_tail_call=no fi])) case "$ac_cv_tail_call" in yes*) - AC_DEFINE([Py_TAIL_CALLING_INTERP], [1], + AC_DEFINE([Py_TAIL_CALL_INTERP], [1], [Define if the C compiler supports efficient proper tail calls.]) esac diff --git a/pyconfig.h.in b/pyconfig.h.in index f1629f659795a1..521dbec058e953 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1719,7 +1719,7 @@ #undef Py_SUNOS_VERSION /* Define if the C compiler supports efficient proper tail calls. */ -#undef Py_TAIL_CALLING_INTERP +#undef Py_TAIL_CALL_INTERP /* Define if you want to enable tracing references for debugging purpose */ #undef Py_TRACE_REFS From fc91ac88480cc1cd8a465ad2b365bc0214b62d1e Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 10 Jan 2025 05:32:22 +0800 Subject: [PATCH 014/110] cleanup macros --- Python/ceval_macros.h | 53 +++++++++++++++++++------------------------ 1 file changed, 23 insertions(+), 30 deletions(-) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 18572df7d7a152..210995df201941 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -71,32 +71,25 @@ #endif #ifdef Py_TAIL_CALL_INTERP -#ifdef LLTRACE -__attribute__((preserve_none)) -typedef PyObject* (*py_tail_call_funcptr)(_PyInterpreterFrame *, _PyStackRef *, PyThreadState *tstate, _Py_CODEUNIT *, int, _PyInterpreterFrame *, int); -#else -__attribute__((preserve_none)) -typedef PyObject* (*py_tail_call_funcptr)(_PyInterpreterFrame *, _PyStackRef *, PyThreadState *tstate, _Py_CODEUNIT *, int, _PyInterpreterFrame *); -#endif -#ifdef LLTRACE -#define DISPATCH_GOTO() do { \ - __attribute__((musttail)) \ - return (INSTRUCTION_TABLE[opcode])(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); \ +# define Py_MUSTTAIL __attribute__((musttail)) +# define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) +# ifdef LLTRACE + Py_PRESERVE_NONE_CC + typedef PyObject* (*py_tail_call_funcptr)(_PyInterpreterFrame *, _PyStackRef *, PyThreadState *tstate, _Py_CODEUNIT *, int, _PyInterpreterFrame *, int); +# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace +# else + Py_PRESERVE_NONE_CC + typedef PyObject* (*py_tail_call_funcptr)(_PyInterpreterFrame *, _PyStackRef *, PyThreadState *tstate, _Py_CODEUNIT *, int, _PyInterpreterFrame *); +# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, oparg, entry_frame +# endif +# define DISPATCH_GOTO() do { \ + Py_MUSTTAIL \ + return (INSTRUCTION_TABLE[opcode])(TAIL_CALL_ARGS); \ } while (0) -#define CEVAL_GOTO(name) do { \ - __attribute__((musttail)) \ - return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); \ +# define CEVAL_GOTO(name) do { \ + Py_MUSTTAIL \ + return (_TAIL_CALL_##name)(TAIL_CALL_ARGS); \ } while (0) -#else -#define DISPATCH_GOTO() do { \ - __attribute__((musttail)) \ - return (INSTRUCTION_TABLE[opcode])(frame, stack_pointer, tstate, next_instr, oparg, entry_frame); \ -} while (0) -#define CEVAL_GOTO(name) do { \ - __attribute__((musttail)) \ - return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, next_instr, oparg, entry_frame); \ -} while (0) -#endif #elif USE_COMPUTED_GOTOS # define TARGET(op) TARGET_##op: # define DISPATCH_GOTO() goto *opcode_targets[opcode] @@ -170,8 +163,8 @@ do { \ CEVAL_GOTO(exit_unwind); \ } \ NEXTOPARG(); \ - __attribute__((musttail)) \ - return (INSTRUCTION_TABLE[opcode])(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); \ + Py_MUSTTAIL \ + return (INSTRUCTION_TABLE[opcode])(TAIL_CALL_ARGS); \ } while (0) #else #define DISPATCH_INLINED(NEW_FRAME) \ @@ -187,8 +180,8 @@ do { \ next_instr = frame->instr_ptr; \ stack_pointer = _PyFrame_GetStackPointer(frame); \ NEXTOPARG(); \ - __attribute__((musttail)) \ - return (INSTRUCTION_TABLE[opcode])(frame, stack_pointer, tstate, next_instr, oparg, entry_frame); \ + Py_MUSTTAIL \ + return (INSTRUCTION_TABLE[opcode])(TAIL_CALL_ARGS); \ } while (0) #endif #else @@ -333,12 +326,12 @@ GETITEM(PyObject *v, Py_ssize_t i) { #ifdef Py_TAIL_CALL_INTERP #ifdef LLTRACE #define GO_TO_INSTRUCTION(op) do { \ - __attribute__((musttail)) \ + Py_MUSTTAIL \ return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - _PyOpcode_Caches[_PyOpcode_Deopt[op]], oparg, entry_frame, lltrace); \ } while (0) #else #define GO_TO_INSTRUCTION(op) do { \ - __attribute__((musttail)) \ + Py_MUSTTAIL \ return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - _PyOpcode_Caches[_PyOpcode_Deopt[op]], oparg, entry_frame); \ } while (0) #endif From 982c51d05ad6b6bdc9d2f9ce12542f91e2810ffe Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 10 Jan 2025 05:40:50 +0800 Subject: [PATCH 015/110] Make deopt more efficient --- Python/bytecodes.c | 8 +- Python/ceval_macros.h | 12 +- Python/generated_cases.c.h | 450 ++++++++++----------- Python/generated_tail_call_handlers.c.h | 450 ++++++++++----------- Tools/cases_generator/generators_common.py | 1 + 5 files changed, 461 insertions(+), 460 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 97d9430a96749b..6dffc016dcb252 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -45,7 +45,7 @@ #include "ceval_macros.h" /* Flow control macros */ -#define GO_TO_INSTRUCTION(instname) ((void)0) +#define GO_TO_INSTRUCTION(instname, SIZE) ((void)0) #define inst(name, ...) case name: #define op(name, ...) /* NAME is ignored */ @@ -1942,7 +1942,7 @@ dummy_func( // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR, INSTRUCTION_SIZE); } family(LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR) = { @@ -4255,7 +4255,7 @@ dummy_func( frame, this_instr, function, arg); ERROR_IF(err, error); PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(CALL_KW); + GO_TO_INSTRUCTION(CALL_KW, INSTRUCTION_SIZE); } op(_MAYBE_EXPAND_METHOD_KW, (callable[1], self_or_null[1], args[oparg], kwnames_in -- func[1], maybe_self[1], args[oparg], kwnames_out)) { @@ -4488,7 +4488,7 @@ dummy_func( _CHECK_PERIODIC; inst(INSTRUMENTED_CALL_FUNCTION_EX, ( -- )) { - GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + GO_TO_INSTRUCTION(CALL_FUNCTION_EX, INSTRUCTION_SIZE); } op(_MAKE_CALLARGS_A_TUPLE, (func, unused, callargs, kwargs_in if (oparg & 1) -- func, unused, tuple, kwargs_out if (oparg & 1))) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 210995df201941..986427cb8bd4b6 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -325,14 +325,14 @@ GETITEM(PyObject *v, Py_ssize_t i) { PyStackRef_XCLOSE(tmp); } while (0) #ifdef Py_TAIL_CALL_INTERP #ifdef LLTRACE -#define GO_TO_INSTRUCTION(op) do { \ +#define GO_TO_INSTRUCTION(op, SIZE) do { \ Py_MUSTTAIL \ - return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - _PyOpcode_Caches[_PyOpcode_Deopt[op]], oparg, entry_frame, lltrace); \ + return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - SIZE, oparg, entry_frame, lltrace); \ } while (0) #else -#define GO_TO_INSTRUCTION(op) do { \ +#define GO_TO_INSTRUCTION(op, SIZE) do { \ Py_MUSTTAIL \ - return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - _PyOpcode_Caches[_PyOpcode_Deopt[op]], oparg, entry_frame); \ + return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - SIZE, oparg, entry_frame); \ } while (0) #endif #else @@ -353,12 +353,12 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define UPDATE_MISS_STATS(INSTNAME) ((void)0) #endif -#define DEOPT_IF(COND, INSTNAME) \ +#define DEOPT_IF(COND, INSTNAME, SIZE) \ if ((COND)) { \ /* This is only a single jump on release builds! */ \ UPDATE_MISS_STATS((INSTNAME)); \ /* assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); */ \ - GO_TO_INSTRUCTION(INSTNAME); \ + GO_TO_INSTRUCTION(INSTNAME, SIZE); \ } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 439f1424f3465e..9dc2c98418a4c3 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -72,8 +72,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_ADD_FLOAT @@ -108,8 +108,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_ADD_INT @@ -143,8 +143,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_ADD_UNICODE @@ -177,8 +177,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_INPLACE_ADD_UNICODE @@ -193,7 +193,7 @@ next_oparg = CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); - DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP); + DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -239,8 +239,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_MULTIPLY_FLOAT @@ -275,8 +275,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_MULTIPLY_INT @@ -310,8 +310,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_SUBTRACT_FLOAT @@ -346,8 +346,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_SUBTRACT_INT @@ -477,7 +477,7 @@ dict_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -511,22 +511,22 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); } // _BINARY_SUBSCR_CHECK_FUNC { container = stack_pointer[-2]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR); + DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); assert(PyFunction_Check(getitem_o)); uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR); + DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); } @@ -571,19 +571,19 @@ list_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; #ifdef Py_GIL_DISABLED _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); stack_pointer = _PyFrame_GetStackPointer(frame); - DEOPT_IF(res_o == NULL, BINARY_SUBSCR); + DEOPT_IF(res_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); #else - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); @@ -611,14 +611,14 @@ str_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); @@ -643,12 +643,12 @@ tuple_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); @@ -1033,7 +1033,7 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_AND_ALLOCATE_OBJECT { @@ -1044,17 +1044,17 @@ self = &stack_pointer[-1 - oparg]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(!PyType_Check(callable_o), CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL, INLINE_CACHE_ENTRIES_CALL); assert(tp->tp_new == PyBaseObject_Type.tp_new); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); assert(tp->tp_alloc == PyType_GenericAlloc); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *self_o = PyType_GenericAlloc(tp, 0); @@ -1131,14 +1131,14 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS { null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { @@ -1157,9 +1157,9 @@ callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_EXACT_ARGS { @@ -1168,15 +1168,15 @@ assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_PY_EXACT_ARGS { @@ -1234,7 +1234,7 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_METHOD_VERSION { @@ -1242,11 +1242,11 @@ callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + DEOPT_IF(!PyFunction_Check(func), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); } // _EXPAND_METHOD { @@ -1340,9 +1340,9 @@ args--; total_args++; } - DEOPT_IF(!PyType_Check(callable_o), CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + DEOPT_IF(tp->tp_vectorcall == NULL, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { @@ -1418,8 +1418,8 @@ args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ @@ -1501,8 +1501,8 @@ args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1585,11 +1585,11 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL); + DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; @@ -1860,9 +1860,9 @@ args--; total_args++; } - DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL); + DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyStackRef cls_stackref = args[1]; _PyStackRef inst_stackref = args[0]; @@ -2056,7 +2056,7 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + DEOPT_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CHECK_METHOD_VERSION_KW { @@ -2064,11 +2064,11 @@ callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL_KW); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW); + DEOPT_IF(!PyFunction_Check(func), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _EXPAND_METHOD_KW { @@ -2161,8 +2161,8 @@ { callable = &stack_pointer[-3 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL_KW); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW); + DEOPT_IF(PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CALL_KW_NON_PY { @@ -2250,16 +2250,16 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + DEOPT_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CHECK_FUNCTION_VERSION_KW { callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW); + DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL_KW); + DEOPT_IF(func->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _PY_FRAME_KW { @@ -2342,9 +2342,9 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.len, CALL); + DEOPT_IF(callable_o != interp->callable_cache.len, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyStackRef arg_stackref = args[0]; PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); @@ -2385,10 +2385,10 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL); + DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL, INLINE_CACHE_ENTRIES_CALL); assert(self_o != NULL); - DEOPT_IF(!PyList_Check(self_o), CALL); - DEOPT_IF(!LOCK_OBJECT(self_o), CALL); + DEOPT_IF(!PyList_Check(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!LOCK_OBJECT(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); @@ -2430,11 +2430,11 @@ } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(args, total_args, args_o); @@ -2514,12 +2514,12 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *d_type = method->d_common.d_type; PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(args, total_args, args_o); @@ -2599,16 +2599,16 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; _PyStackRef self_stackref = args[0]; PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -2671,16 +2671,16 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); + DEOPT_IF(meth->ml_flags != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); _PyStackRef arg_stackref = args[1]; _PyStackRef self_stackref = args[0]; DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type), CALL); + method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -2738,8 +2738,8 @@ { callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL); + DEOPT_IF(PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CALL_NON_PY_GENERAL { @@ -2821,16 +2821,16 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_EXACT_ARGS { @@ -2839,15 +2839,15 @@ assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_PY_EXACT_ARGS { @@ -2902,16 +2902,16 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _PY_FRAME_GENERAL { @@ -2987,8 +2987,8 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Str(arg_o); @@ -3038,8 +3038,8 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PySequence_Tuple(arg_o); @@ -3087,8 +3087,8 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); PyStackRef_CLOSE(arg); @@ -3290,8 +3290,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_FLOAT @@ -3328,16 +3328,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_INT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && _PyLong_DigitCount((PyLongObject *)right_o) <= 1); @@ -3370,8 +3370,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_STR @@ -3454,7 +3454,7 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP); + DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); @@ -3482,7 +3482,7 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP); + DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3985,14 +3985,14 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _FOR_ITER_GEN_FRAME { iter = stack_pointer[-1]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); STAT_INC(FOR_ITER, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, PyStackRef_None); @@ -4033,7 +4033,7 @@ // _ITER_CHECK_LIST { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_LIST { @@ -4085,7 +4085,7 @@ { iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_RANGE { @@ -4129,7 +4129,7 @@ // _ITER_CHECK_TUPLE { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_TUPLE { @@ -4546,7 +4546,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + GO_TO_INSTRUCTION(CALL_FUNCTION_EX, 1 ); } TARGET(INSTRUMENTED_CALL_KW) { @@ -4570,7 +4570,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (err) CEVAL_GOTO(error); PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(CALL_KW); + GO_TO_INSTRUCTION(CALL_KW, 4 ); } TARGET(INSTRUMENTED_END_FOR) { @@ -4762,7 +4762,7 @@ // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR, 2 ); } TARGET(INSTRUMENTED_NOT_TAKEN) { @@ -5288,9 +5288,9 @@ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS @@ -5323,16 +5323,16 @@ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_TYPE_VERSION { uint32_t type_version = read_u32(&this_instr[4].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_CLASS { @@ -5363,17 +5363,17 @@ PyObject *getattribute = read_obj(&this_instr[6].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( @@ -5401,14 +5401,14 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_MANAGED_OBJECT_HAS_VALUES { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { @@ -5416,7 +5416,7 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = *value_ptr; - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr_o); null = PyStackRef_NULL; @@ -5446,7 +5446,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_ATTR_METHOD_LAZY_DICT { @@ -5454,7 +5454,7 @@ char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = *(PyObject **)ptr; /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR); + DEOPT_IF(dict != NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 1 cache entry */ // _LOAD_ATTR_METHOD_LAZY_DICT @@ -5489,7 +5489,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_NO_DICT @@ -5525,20 +5525,20 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&this_instr[4].cache); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_METHOD_WITH_VALUES { @@ -5573,11 +5573,11 @@ owner = stack_pointer[-1]; uint32_t dict_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR); + DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); mod_keys = keys; } // _LOAD_ATTR_MODULE_FROM_KEYS @@ -5588,11 +5588,11 @@ PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); // Clear mod_keys from stack in case we need to deopt - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); if (!increfed) { - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } #else Py_INCREF(attr_o); @@ -5624,7 +5624,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT @@ -5655,20 +5655,20 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&this_instr[4].cache); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { @@ -5693,7 +5693,7 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_TYPE_VERSION { @@ -5701,7 +5701,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_PROPERTY_FRAME @@ -5711,10 +5711,10 @@ assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; PyCodeObject *code = (PyCodeObject *)f->func_code; - DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR); - DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR); - DEOPT_IF(code->co_argcount != 1, LOAD_ATTR); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(code->co_argcount != 1, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); new_frame->localsplus[0] = owner; @@ -5763,7 +5763,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_SLOT { @@ -5771,7 +5771,7 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); char *addr = (char *)owner_o + index; PyObject *attr_o = *(PyObject **)addr; - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; attr = PyStackRef_FromPyObjectNew(attr_o); @@ -5800,14 +5800,14 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_ATTR_WITH_HINT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, LOAD_ATTR); + DEOPT_IF(dict == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); } // _LOAD_ATTR_WITH_HINT @@ -5816,13 +5816,13 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject *attr_o; PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), LOAD_ATTR); + DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); + DEOPT_IF(ep->me_key != name, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); attr_o = ep->me_value; - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr_o); attr = PyStackRef_FromPyObjectSteal(attr_o); @@ -6160,18 +6160,18 @@ { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); assert(DK_IS_UNICODE(keys)); } // _GUARD_BUILTINS_VERSION_PUSH_KEYS { uint16_t version = read_u16(&this_instr[3].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); builtins_keys = keys; assert(DK_IS_UNICODE(builtins_keys)); } @@ -6180,10 +6180,10 @@ uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL); + DEOPT_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -6211,9 +6211,9 @@ { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); globals_keys = keys; assert(DK_IS_UNICODE(globals_keys)); } @@ -6223,10 +6223,10 @@ uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL); + DEOPT_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -6445,8 +6445,8 @@ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6481,8 +6481,8 @@ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -6998,16 +6998,16 @@ INSTRUCTION_STATS(RESUME_CHECK); static_assert(0 == 0, "incorrect cache size"); #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME, 0); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); + DEOPT_IF(eval_breaker != version, RESUME, 0); #ifdef Py_GIL_DISABLED DEOPT_IF(frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME); + ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME, 0); #endif DISPATCH(); } @@ -7177,15 +7177,15 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, SEND); + DEOPT_IF(tstate->interp->eval_frame, SEND, INLINE_CACHE_ENTRIES_SEND); } // _SEND_GEN_FRAME { v = stack_pointer[-1]; receiver = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND, INLINE_CACHE_ENTRIES_SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND, INLINE_CACHE_ENTRIES_SEND); STAT_INC(SEND, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, v); @@ -7373,11 +7373,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); PyTypeObject *tp = Py_TYPE(owner_o); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } } // _GUARD_DORV_NO_DICT @@ -7388,7 +7388,7 @@ if (_PyObject_GetManagedDict(owner_o) || !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } } // _STORE_ATTR_INSTANCE_VALUE @@ -7429,14 +7429,14 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } // _STORE_ATTR_SLOT { value = stack_pointer[-2]; uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; @@ -7464,7 +7464,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } // _STORE_ATTR_WITH_HINT { @@ -7473,12 +7473,12 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, STORE_ATTR); - DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR); + DEOPT_IF(dict == NULL, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); #ifdef Py_GIL_DISABLED if (dict != _PyObject_GetManagedDict(owner_o)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } #endif assert(PyDict_CheckExact((PyObject *)dict)); @@ -7486,17 +7486,17 @@ if (hint >= (size_t)dict->ma_keys->dk_nentries || !DK_IS_UNICODE(dict->ma_keys)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } PyObject *old_value = ep->me_value; if (old_value == NULL) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } _PyFrame_SetStackPointer(frame, stack_pointer); _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); @@ -7730,7 +7730,7 @@ dict_st = stack_pointer[-2]; value = stack_pointer[-3]; PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, @@ -7758,16 +7758,16 @@ value = stack_pointer[-3]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR); + DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); // Ensure index < len(list) if (index >= PyList_GET_SIZE(list)) { UNLOCK_OBJECT(list); - DEOPT_IF(true, STORE_SUBSCR); + DEOPT_IF(true, STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); } STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); @@ -7855,7 +7855,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); } // _REPLACE_WITH_TRUE { @@ -7876,7 +7876,7 @@ /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; - DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL); + DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); DISPATCH(); } @@ -7892,7 +7892,7 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL); + DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value_o)) { assert(_Py_IsImmortal(value_o)); @@ -7917,7 +7917,7 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL); + DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; PyStackRef_CLOSE(value); @@ -7936,7 +7936,7 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL); + DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); res = PyStackRef_False; stack_pointer[-1] = res; @@ -7954,7 +7954,7 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL); + DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); if (value_o == &_Py_STR(empty)) { assert(_Py_IsImmortal(value_o)); @@ -8090,11 +8090,11 @@ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); if (PyList_GET_SIZE(seq_o) != oparg) { UNLOCK_OBJECT(seq_o); - DEOPT_IF(true, UNPACK_SEQUENCE); + DEOPT_IF(true, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq_o); @@ -8119,8 +8119,8 @@ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { @@ -8144,8 +8144,8 @@ seq = stack_pointer[-1]; assert(oparg == 2); PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 46448980b3c238..8621021b045440 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -368,8 +368,8 @@ _TAIL_CALL_BINARY_OP_ADD_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_po left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_ADD_FLOAT @@ -418,8 +418,8 @@ _TAIL_CALL_BINARY_OP_ADD_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_poin left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_ADD_INT @@ -467,8 +467,8 @@ _TAIL_CALL_BINARY_OP_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef *stack_ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_ADD_UNICODE @@ -515,8 +515,8 @@ _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_INPLACE_ADD_UNICODE @@ -531,7 +531,7 @@ _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef next_oparg = CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); - DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP); + DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -591,8 +591,8 @@ _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *sta left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_MULTIPLY_FLOAT @@ -641,8 +641,8 @@ _TAIL_CALL_BINARY_OP_MULTIPLY_INT(_PyInterpreterFrame *frame, _PyStackRef *stack left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_MULTIPLY_INT @@ -690,8 +690,8 @@ _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *sta left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_SUBTRACT_FLOAT @@ -740,8 +740,8 @@ _TAIL_CALL_BINARY_OP_SUBTRACT_INT(_PyInterpreterFrame *frame, _PyStackRef *stack left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_SUBTRACT_INT @@ -913,7 +913,7 @@ _TAIL_CALL_BINARY_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_poi dict_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -961,22 +961,22 @@ _TAIL_CALL_BINARY_SUBSCR_GETITEM(_PyInterpreterFrame *frame, _PyStackRef *stack_ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); } // _BINARY_SUBSCR_CHECK_FUNC { container = stack_pointer[-2]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR); + DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); assert(PyFunction_Check(getitem_o)); uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR); + DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); } @@ -1035,19 +1035,19 @@ _TAIL_CALL_BINARY_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack list_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; #ifdef Py_GIL_DISABLED _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); stack_pointer = _PyFrame_GetStackPointer(frame); - DEOPT_IF(res_o == NULL, BINARY_SUBSCR); + DEOPT_IF(res_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); #else - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); @@ -1089,14 +1089,14 @@ _TAIL_CALL_BINARY_SUBSCR_STR_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_ str_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); @@ -1135,12 +1135,12 @@ _TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(_PyInterpreterFrame *frame, _PyStackRef *stac tuple_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); @@ -1651,7 +1651,7 @@ _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(_PyInterpreterFrame *frame, _PyStackRef *st /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_AND_ALLOCATE_OBJECT { @@ -1662,17 +1662,17 @@ _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(_PyInterpreterFrame *frame, _PyStackRef *st self = &stack_pointer[-1 - oparg]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(!PyType_Check(callable_o), CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL, INLINE_CACHE_ENTRIES_CALL); assert(tp->tp_new == PyBaseObject_Type.tp_new); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); assert(tp->tp_alloc == PyType_GenericAlloc); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *self_o = PyType_GenericAlloc(tp, 0); @@ -1763,14 +1763,14 @@ _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS { null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { @@ -1789,9 +1789,9 @@ _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_EXACT_ARGS { @@ -1800,15 +1800,15 @@ _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_PY_EXACT_ARGS { @@ -1880,7 +1880,7 @@ _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *st /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_METHOD_VERSION { @@ -1888,11 +1888,11 @@ _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *st callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + DEOPT_IF(!PyFunction_Check(func), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); } // _EXPAND_METHOD { @@ -2000,9 +2000,9 @@ _TAIL_CALL_CALL_BUILTIN_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_poi args--; total_args++; } - DEOPT_IF(!PyType_Check(callable_o), CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + DEOPT_IF(tp->tp_vectorcall == NULL, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { @@ -2092,8 +2092,8 @@ _TAIL_CALL_CALL_BUILTIN_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_poin args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ @@ -2189,8 +2189,8 @@ _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackR args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2287,11 +2287,11 @@ _TAIL_CALL_CALL_BUILTIN_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL); + DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; @@ -2618,9 +2618,9 @@ _TAIL_CALL_CALL_ISINSTANCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe args--; total_args++; } - DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL); + DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyStackRef cls_stackref = args[1]; _PyStackRef inst_stackref = args[0]; @@ -2842,7 +2842,7 @@ _TAIL_CALL_CALL_KW_BOUND_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_p /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + DEOPT_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CHECK_METHOD_VERSION_KW { @@ -2850,11 +2850,11 @@ _TAIL_CALL_CALL_KW_BOUND_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_p callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL_KW); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW); + DEOPT_IF(!PyFunction_Check(func), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _EXPAND_METHOD_KW { @@ -2961,8 +2961,8 @@ _TAIL_CALL_CALL_KW_NON_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer { callable = &stack_pointer[-3 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL_KW); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW); + DEOPT_IF(PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CALL_KW_NON_PY { @@ -3064,16 +3064,16 @@ _TAIL_CALL_CALL_KW_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + DEOPT_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CHECK_FUNCTION_VERSION_KW { callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW); + DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL_KW); + DEOPT_IF(func->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _PY_FRAME_KW { @@ -3170,9 +3170,9 @@ _TAIL_CALL_CALL_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.len, CALL); + DEOPT_IF(callable_o != interp->callable_cache.len, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyStackRef arg_stackref = args[0]; PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); @@ -3227,10 +3227,10 @@ _TAIL_CALL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_point PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL); + DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL, INLINE_CACHE_ENTRIES_CALL); assert(self_o != NULL); - DEOPT_IF(!PyList_Check(self_o), CALL); - DEOPT_IF(!LOCK_OBJECT(self_o), CALL); + DEOPT_IF(!PyList_Check(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!LOCK_OBJECT(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); @@ -3286,11 +3286,11 @@ _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(_PyInterpreterFrame *frame, _PyStackRef * } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(args, total_args, args_o); @@ -3384,12 +3384,12 @@ _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *d_type = method->d_common.d_type; PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(args, total_args, args_o); @@ -3483,16 +3483,16 @@ _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(_PyInterpreterFrame *frame, _PyStackRef args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; _PyStackRef self_stackref = args[0]; PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -3569,16 +3569,16 @@ _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(_PyInterpreterFrame *frame, _PyStackRef *sta total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); + DEOPT_IF(meth->ml_flags != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); _PyStackRef arg_stackref = args[1]; _PyStackRef self_stackref = args[0]; DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type), CALL); + method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -3650,8 +3650,8 @@ _TAIL_CALL_CALL_NON_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_po { callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL); + DEOPT_IF(PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CALL_NON_PY_GENERAL { @@ -3747,16 +3747,16 @@ _TAIL_CALL_CALL_PY_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_poi /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_EXACT_ARGS { @@ -3765,15 +3765,15 @@ _TAIL_CALL_CALL_PY_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_poi assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_PY_EXACT_ARGS { @@ -3842,16 +3842,16 @@ _TAIL_CALL_CALL_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _PY_FRAME_GENERAL { @@ -3941,8 +3941,8 @@ _TAIL_CALL_CALL_STR_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Str(arg_o); @@ -4006,8 +4006,8 @@ _TAIL_CALL_CALL_TUPLE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PySequence_Tuple(arg_o); @@ -4069,8 +4069,8 @@ _TAIL_CALL_CALL_TYPE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); PyStackRef_CLOSE(arg); @@ -4342,8 +4342,8 @@ _TAIL_CALL_COMPARE_OP_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_point left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_FLOAT @@ -4394,16 +4394,16 @@ _TAIL_CALL_COMPARE_OP_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_INT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && _PyLong_DigitCount((PyLongObject *)right_o) <= 1); @@ -4450,8 +4450,8 @@ _TAIL_CALL_COMPARE_OP_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_STR @@ -4562,7 +4562,7 @@ _TAIL_CALL_CONTAINS_OP_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_point left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP); + DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); @@ -4604,7 +4604,7 @@ _TAIL_CALL_CONTAINS_OP_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP); + DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5401,14 +5401,14 @@ _TAIL_CALL_FOR_ITER_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _FOR_ITER_GEN_FRAME { iter = stack_pointer[-1]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); STAT_INC(FOR_ITER, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, PyStackRef_None); @@ -5463,7 +5463,7 @@ _TAIL_CALL_FOR_ITER_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, // _ITER_CHECK_LIST { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_LIST { @@ -5529,7 +5529,7 @@ _TAIL_CALL_FOR_ITER_RANGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer { iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_RANGE { @@ -5587,7 +5587,7 @@ _TAIL_CALL_FOR_ITER_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer // _ITER_CHECK_TUPLE { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_TUPLE { @@ -6144,7 +6144,7 @@ _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + GO_TO_INSTRUCTION(CALL_FUNCTION_EX, 1 ); } } @@ -6182,7 +6182,7 @@ _TAIL_CALL_INSTRUMENTED_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_p stack_pointer = _PyFrame_GetStackPointer(frame); if (err) CEVAL_GOTO(error); PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(CALL_KW); + GO_TO_INSTRUCTION(CALL_KW, 4 ); } } @@ -6486,7 +6486,7 @@ _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR, 2 ); } } @@ -7250,9 +7250,9 @@ _TAIL_CALL_LOAD_ATTR_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS @@ -7299,16 +7299,16 @@ _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(_PyInterpreterFrame *frame, _PyS owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_TYPE_VERSION { uint32_t type_version = read_u32(&this_instr[4].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_CLASS { @@ -7353,17 +7353,17 @@ _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(_PyInterpreterFrame *frame, _PyStac PyObject *getattribute = read_obj(&this_instr[6].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( @@ -7405,14 +7405,14 @@ _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *sta uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_MANAGED_OBJECT_HAS_VALUES { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { @@ -7420,7 +7420,7 @@ _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *sta PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = *value_ptr; - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr_o); null = PyStackRef_NULL; @@ -7464,7 +7464,7 @@ _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(_PyInterpreterFrame *frame, _PyStackRef *s uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_ATTR_METHOD_LAZY_DICT { @@ -7472,7 +7472,7 @@ _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(_PyInterpreterFrame *frame, _PyStackRef *s char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = *(PyObject **)ptr; /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR); + DEOPT_IF(dict != NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 1 cache entry */ // _LOAD_ATTR_METHOD_LAZY_DICT @@ -7521,7 +7521,7 @@ _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *sta uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_NO_DICT @@ -7571,20 +7571,20 @@ _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&this_instr[4].cache); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_METHOD_WITH_VALUES { @@ -7633,11 +7633,11 @@ _TAIL_CALL_LOAD_ATTR_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_point owner = stack_pointer[-1]; uint32_t dict_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR); + DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); mod_keys = keys; } // _LOAD_ATTR_MODULE_FROM_KEYS @@ -7648,11 +7648,11 @@ _TAIL_CALL_LOAD_ATTR_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_point PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); // Clear mod_keys from stack in case we need to deopt - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); if (!increfed) { - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } #else Py_INCREF(attr_o); @@ -7698,7 +7698,7 @@ _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(_PyInterpreterFrame *frame, _PyStackR uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT @@ -7743,20 +7743,20 @@ _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(_PyInterpreterFrame *frame, _PySt uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&this_instr[4].cache); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { @@ -7795,7 +7795,7 @@ _TAIL_CALL_LOAD_ATTR_PROPERTY(_PyInterpreterFrame *frame, _PyStackRef *stack_poi /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_TYPE_VERSION { @@ -7803,7 +7803,7 @@ _TAIL_CALL_LOAD_ATTR_PROPERTY(_PyInterpreterFrame *frame, _PyStackRef *stack_poi uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_PROPERTY_FRAME @@ -7813,10 +7813,10 @@ _TAIL_CALL_LOAD_ATTR_PROPERTY(_PyInterpreterFrame *frame, _PyStackRef *stack_poi assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; PyCodeObject *code = (PyCodeObject *)f->func_code; - DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR); - DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR); - DEOPT_IF(code->co_argcount != 1, LOAD_ATTR); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(code->co_argcount != 1, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); new_frame->localsplus[0] = owner; @@ -7879,7 +7879,7 @@ _TAIL_CALL_LOAD_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_SLOT { @@ -7887,7 +7887,7 @@ _TAIL_CALL_LOAD_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); char *addr = (char *)owner_o + index; PyObject *attr_o = *(PyObject **)addr; - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; attr = PyStackRef_FromPyObjectNew(attr_o); @@ -7930,14 +7930,14 @@ _TAIL_CALL_LOAD_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_po uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_ATTR_WITH_HINT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, LOAD_ATTR); + DEOPT_IF(dict == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); } // _LOAD_ATTR_WITH_HINT @@ -7946,13 +7946,13 @@ _TAIL_CALL_LOAD_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_po PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject *attr_o; PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), LOAD_ATTR); + DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); + DEOPT_IF(ep->me_key != name, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); attr_o = ep->me_value; - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr_o); attr = PyStackRef_FromPyObjectSteal(attr_o); @@ -8472,18 +8472,18 @@ _TAIL_CALL_LOAD_GLOBAL_BUILTIN(_PyInterpreterFrame *frame, _PyStackRef *stack_po { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); assert(DK_IS_UNICODE(keys)); } // _GUARD_BUILTINS_VERSION_PUSH_KEYS { uint16_t version = read_u16(&this_instr[3].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); builtins_keys = keys; assert(DK_IS_UNICODE(builtins_keys)); } @@ -8492,10 +8492,10 @@ _TAIL_CALL_LOAD_GLOBAL_BUILTIN(_PyInterpreterFrame *frame, _PyStackRef *stack_po uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL); + DEOPT_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -8537,9 +8537,9 @@ _TAIL_CALL_LOAD_GLOBAL_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_poi { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); globals_keys = keys; assert(DK_IS_UNICODE(globals_keys)); } @@ -8549,10 +8549,10 @@ _TAIL_CALL_LOAD_GLOBAL_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_poi uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL); + DEOPT_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -8855,8 +8855,8 @@ _TAIL_CALL_LOAD_SUPER_ATTR_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_p PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -8905,8 +8905,8 @@ _TAIL_CALL_LOAD_SUPER_ATTR_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -9730,16 +9730,16 @@ _TAIL_CALL_RESUME_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, INSTRUCTION_STATS(RESUME_CHECK); static_assert(0 == 0, "incorrect cache size"); #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME, 0); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); + DEOPT_IF(eval_breaker != version, RESUME, 0); #ifdef Py_GIL_DISABLED DEOPT_IF(frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME); + ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME, 0); #endif DISPATCH(); } @@ -9965,15 +9965,15 @@ _TAIL_CALL_SEND_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, SEND); + DEOPT_IF(tstate->interp->eval_frame, SEND, INLINE_CACHE_ENTRIES_SEND); } // _SEND_GEN_FRAME { v = stack_pointer[-1]; receiver = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND, INLINE_CACHE_ENTRIES_SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND, INLINE_CACHE_ENTRIES_SEND); STAT_INC(SEND, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, v); @@ -10245,11 +10245,11 @@ _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *st uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); PyTypeObject *tp = Py_TYPE(owner_o); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } } // _GUARD_DORV_NO_DICT @@ -10260,7 +10260,7 @@ _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *st if (_PyObject_GetManagedDict(owner_o) || !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } } // _STORE_ATTR_INSTANCE_VALUE @@ -10315,14 +10315,14 @@ _TAIL_CALL_STORE_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } // _STORE_ATTR_SLOT { value = stack_pointer[-2]; uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; @@ -10364,7 +10364,7 @@ _TAIL_CALL_STORE_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_p uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } // _STORE_ATTR_WITH_HINT { @@ -10373,12 +10373,12 @@ _TAIL_CALL_STORE_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_p PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, STORE_ATTR); - DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR); + DEOPT_IF(dict == NULL, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); #ifdef Py_GIL_DISABLED if (dict != _PyObject_GetManagedDict(owner_o)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } #endif assert(PyDict_CheckExact((PyObject *)dict)); @@ -10386,17 +10386,17 @@ _TAIL_CALL_STORE_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_p if (hint >= (size_t)dict->ma_keys->dk_nentries || !DK_IS_UNICODE(dict->ma_keys)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } PyObject *old_value = ep->me_value; if (old_value == NULL) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } _PyFrame_SetStackPointer(frame, stack_pointer); _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); @@ -10756,7 +10756,7 @@ _TAIL_CALL_STORE_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_poin dict_st = stack_pointer[-2]; value = stack_pointer[-3]; PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, @@ -10798,16 +10798,16 @@ _TAIL_CALL_STORE_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_ value = stack_pointer[-3]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR); + DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); // Ensure index < len(list) if (index >= PyList_GET_SIZE(list)) { UNLOCK_OBJECT(list); - DEOPT_IF(true, STORE_SUBSCR); + DEOPT_IF(true, STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); } STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); @@ -10937,7 +10937,7 @@ _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_po uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); } // _REPLACE_WITH_TRUE { @@ -10972,7 +10972,7 @@ _TAIL_CALL_TO_BOOL_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; - DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL); + DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); DISPATCH(); } @@ -11002,7 +11002,7 @@ _TAIL_CALL_TO_BOOL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL); + DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value_o)) { assert(_Py_IsImmortal(value_o)); @@ -11041,7 +11041,7 @@ _TAIL_CALL_TO_BOOL_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL); + DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; PyStackRef_CLOSE(value); @@ -11074,7 +11074,7 @@ _TAIL_CALL_TO_BOOL_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, /* Skip 2 cache entries */ value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL); + DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); res = PyStackRef_False; stack_pointer[-1] = res; @@ -11106,7 +11106,7 @@ _TAIL_CALL_TO_BOOL_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL); + DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); if (value_o == &_Py_STR(empty)) { assert(_Py_IsImmortal(value_o)); @@ -11326,11 +11326,11 @@ _TAIL_CALL_UNPACK_SEQUENCE_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_p seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); if (PyList_GET_SIZE(seq_o) != oparg) { UNLOCK_OBJECT(seq_o); - DEOPT_IF(true, UNPACK_SEQUENCE); + DEOPT_IF(true, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq_o); @@ -11369,8 +11369,8 @@ _TAIL_CALL_UNPACK_SEQUENCE_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { @@ -11408,8 +11408,8 @@ _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *st seq = stack_pointer[-1]; assert(oparg == 2); PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 30af5beb51e010..64098cf5d9332a 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -154,6 +154,7 @@ def deopt_if( assert inst is not None assert inst.family is not None self.out.emit(inst.family.name) + self.out.emit(f", {inst.family.size}") self.out.emit(");\n") return not always_true(first_tkn) From 98203405058f8336036109bb15ace32b658a6731 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 10 Jan 2025 05:45:51 +0800 Subject: [PATCH 016/110] Cleanup --- Python/ceval.c | 18 +++++++++--------- .../tier1_tail_call_generator.py | 4 +--- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index f1995751ef3cf7..a842c3e13311c0 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -821,8 +821,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int int lltrace = 0; #endif - _PyInterpreterFrame e; - _PyInterpreterFrame *entry_frame = &e; + _PyInterpreterFrame entry_f; + _PyInterpreterFrame *entry_frame = &entry_f; @@ -834,14 +834,14 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int e.f_globals = (PyObject*)0xaaa3; e.f_builtins = (PyObject*)0xaaa4; #endif - e.f_executable = PyStackRef_None; - e.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; - e.stackpointer = e.localsplus; - e.owner = FRAME_OWNED_BY_CSTACK; - e.visited = 0; - e.return_offset = 0; + entry_f.f_executable = PyStackRef_None; + entry_f.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; + entry_f.stackpointer = entry_f.localsplus; + entry_f.owner = FRAME_OWNED_BY_CSTACK; + entry_f.visited = 0; + entry_f.return_offset = 0; /* Push frame */ - e.previous = tstate->current_frame; + entry_f.previous = tstate->current_frame; frame->previous = entry_frame; tstate->current_frame = frame; diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index 68164c3cb62d63..59fe06735dac28 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -4,7 +4,6 @@ """ import argparse -import re from typing import TextIO @@ -60,7 +59,7 @@ def error_handler(out: CWriter, body: str, next: str, emit_tail_call: bool = Tru def generate_tier1( - filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool + filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool ) -> None: write_header(__file__, filenames, outfile) outfile.write( @@ -217,7 +216,6 @@ def generate_tier1( out.emit("\n") out.emit(function_proto(name)) out.emit("{\n") - # out.emit(f'printf("{name}\\n");\n') out.emit("int opcode = next_instr->op.code;\n") out.emit("{\n") write_single_inst(out, emitter, name, inst) From 5862338076927e24175d34c4f3e90d647d099e3e Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 10 Jan 2025 06:56:49 +0800 Subject: [PATCH 017/110] more cleanup --- Python/ceval.c | 20 ++++---------------- Python/ceval_macros.h | 18 ++++++++++-------- 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index a842c3e13311c0..437af9e2412656 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -783,24 +783,12 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) #ifdef Py_TAIL_CALL_INTERP #include "generated_tail_call_handlers.c.h" -# ifdef LLTRACE - static inline PyObject * - _TAIL_CALL_shim(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) - { - return (INSTRUCTION_TABLE[next_instr->op.code])(frame, stack_pointer, tstate, next_instr, next_instr->op.arg, entry_frame, lltrace); - } -# else - static inline PyObject * - _TAIL_CALL_shim(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) - { - return (INSTRUCTION_TABLE[next_instr->op.code])(frame, stack_pointer, tstate, next_instr, next_instr->op.arg, entry_frame); - } -# endif +static inline PyObject *_TAIL_CALL_shim(TAIL_CALL_PARAMS) +{ + return (INSTRUCTION_TABLE[next_instr->op.code])(frame, stack_pointer, tstate, next_instr, next_instr->op.arg, entry_frame); +} #endif - PyObject* _Py_HOT_FUNCTION _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 986427cb8bd4b6..11dec0412a51bd 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -70,18 +70,19 @@ #define INSTRUCTION_STATS(op) ((void)0) #endif +#ifdef LLTRACE +# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame *entry_frame, int lltrace +# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace +#else +# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame *entry_frame +# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, oparg, entry_frame +#endif + #ifdef Py_TAIL_CALL_INTERP # define Py_MUSTTAIL __attribute__((musttail)) # define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) -# ifdef LLTRACE - Py_PRESERVE_NONE_CC - typedef PyObject* (*py_tail_call_funcptr)(_PyInterpreterFrame *, _PyStackRef *, PyThreadState *tstate, _Py_CODEUNIT *, int, _PyInterpreterFrame *, int); -# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace -# else Py_PRESERVE_NONE_CC - typedef PyObject* (*py_tail_call_funcptr)(_PyInterpreterFrame *, _PyStackRef *, PyThreadState *tstate, _Py_CODEUNIT *, int, _PyInterpreterFrame *); -# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, oparg, entry_frame -# endif + typedef PyObject* (*py_tail_call_funcptr)(TAIL_CALL_PARAMS); # define DISPATCH_GOTO() do { \ Py_MUSTTAIL \ return (INSTRUCTION_TABLE[opcode])(TAIL_CALL_ARGS); \ @@ -90,6 +91,7 @@ Py_MUSTTAIL \ return (_TAIL_CALL_##name)(TAIL_CALL_ARGS); \ } while (0) + #elif USE_COMPUTED_GOTOS # define TARGET(op) TARGET_##op: # define DISPATCH_GOTO() goto *opcode_targets[opcode] From f5b1c93f72143abf0895ea7a266b61ef6fd4a08b Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 10 Jan 2025 08:16:05 +0800 Subject: [PATCH 018/110] Generate everything --- Python/ceval.c | 22 +- Python/ceval_macros.h | 2 + Python/generated_tail_call_handlers.c.h | 3393 +++-------------- .../tier1_tail_call_generator.py | 198 +- 4 files changed, 605 insertions(+), 3010 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 437af9e2412656..3cdea3f3512abf 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -910,7 +910,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DISPATCH(); #endif - { #ifndef Py_TAIL_CALL_INTERP /* Start instructions */ #if !USE_COMPUTED_GOTOS @@ -942,15 +941,15 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int or goto error. */ Py_UNREACHABLE(); #endif -pop_4_error: +TAIL_CALL_TARGET(pop_4_error): STACK_SHRINK(1); -pop_3_error: +TAIL_CALL_TARGET(pop_3_error): STACK_SHRINK(1); -pop_2_error: +TAIL_CALL_TARGET(pop_2_error): STACK_SHRINK(1); -pop_1_error: +TAIL_CALL_TARGET(pop_1_error): STACK_SHRINK(1); -error: +TAIL_CALL_TARGET(error): /* Double-check exception status. */ #ifdef NDEBUG if (!_PyErr_Occurred(tstate)) { @@ -970,7 +969,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } } _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: +TAIL_CALL_TARGET(exception_unwind): { /* We can't use frame->instr_ptr here, as RERAISE may have set it */ int offset = INSTR_OFFSET()-1; @@ -1032,9 +1031,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DISPATCH(); #endif } - } -exit_unwind: +TAIL_CALL_TARGET(exit_unwind): assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); assert(frame != entry_frame); @@ -1050,12 +1048,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int return NULL; } -resume_with_error: +TAIL_CALL_TARGET(resume_with_error): next_instr = frame->instr_ptr; stack_pointer = _PyFrame_GetStackPointer(frame); goto error; - +/* END_BASE_INTERPRETER */ #ifdef _Py_TIER2 // Tier 2 is also here! @@ -1197,7 +1195,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif // _Py_TIER2 -} +} /* _PyEval_EvalFrameDefault */ #ifdef DO_NOT_OPTIMIZE_INTERP_LOOP # pragma optimize("", on) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 11dec0412a51bd..fb07c87977e46a 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -102,6 +102,8 @@ # define CEVAL_GOTO(name) goto name; #endif +#define TAIL_CALL_TARGET(name) name + /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ #ifdef LLTRACE diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 8621021b045440..b2a95865d22814 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -7,290 +7,157 @@ #error "This file is for tail-calling interpreter only." #endif #define TIER_ONE 1 +static inline PyObject *_TAIL_CALL_shim(TAIL_CALL_PARAMS); static py_tail_call_funcptr INSTRUCTION_TABLE[256]; - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -; - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_resume_with_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_resume_with_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_pop_4_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_pop_3_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_resume_with_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_pop_4_error(TAIL_CALL_PARAMS) { - int opcode = next_instr->op.code; - - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - #ifdef LLTRACE - __attribute__((musttail)) - return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); - #else - __attribute__((musttail)) - return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame); - #endif + STACK_SHRINK(1); + CEVAL_GOTO(pop_3_error); } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_exit_unwind(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_exit_unwind(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_pop_3_error(TAIL_CALL_PARAMS) { - int opcode = next_instr->op.code; - - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - #ifdef LLTRACE - __attribute__((musttail)) - return _TAIL_CALL_resume_with_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); + STACK_SHRINK(1); + CEVAL_GOTO(pop_2_error); +} +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS) +{ + STACK_SHRINK(1); + CEVAL_GOTO(pop_1_error); +} +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS) +{ + STACK_SHRINK(1); + CEVAL_GOTO(error); +} +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_error(TAIL_CALL_PARAMS) +{ + /* Double-check exception status. */ + #ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } #else - __attribute__((musttail)) - return _TAIL_CALL_resume_with_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame); + assert(_PyErr_Occurred(tstate)); #endif -} -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_exception_unwind(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_exception_unwind(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif + /* Log traceback info. */ + assert(frame != entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); + CEVAL_GOTO(exception_unwind); +} +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS) { - int opcode = next_instr->op.code; - - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + CEVAL_GOTO(exit_unwind); + } - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - CEVAL_GOTO(exit_unwind); - } + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + CEVAL_GOTO(exception_unwind); + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - CEVAL_GOTO(exception_unwind); - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + CEVAL_GOTO(exception_unwind); + } + /* Resume normal execution */ + #ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } + #endif - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - CEVAL_GOTO(exception_unwind); - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); + #ifdef Py_TAIL_CALL_INTERP + #ifdef LLTRACE + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, entry_frame, lltrace); + #else + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, entry_frame); + #endif + #else + DISPATCH(); + #endif } -#endif - DISPATCH(); - } -} -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif + CEVAL_GOTO(exit_unwind); +} +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS) { - int opcode = next_instr->op.code; - - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ + _Py_LeaveRecursiveCallPy(tstate); assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); - #ifdef LLTRACE - __attribute__((musttail)) - return _TAIL_CALL_exception_unwind(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); - #else - __attribute__((musttail)) - return _TAIL_CALL_exception_unwind(frame, stack_pointer, tstate, next_instr, oparg, entry_frame); - #endif -} - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_pop_1_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_pop_1_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - STACK_SHRINK(1); - #ifdef LLTRACE - __attribute__((musttail)) - return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); - #else - __attribute__((musttail)) - return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame); - #endif -} -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_pop_2_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_pop_2_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - STACK_SHRINK(1); - #ifdef LLTRACE - __attribute__((musttail)) - return _TAIL_CALL_pop_1_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); - #else - __attribute__((musttail)) - return _TAIL_CALL_pop_1_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame); - #endif + CEVAL_GOTO(resume_with_error); } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_pop_3_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_pop_3_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_resume_with_error(TAIL_CALL_PARAMS) { - int opcode = next_instr->op.code; - STACK_SHRINK(1); - #ifdef LLTRACE - __attribute__((musttail)) - return _TAIL_CALL_pop_2_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); - #else - __attribute__((musttail)) - return _TAIL_CALL_pop_2_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame); - #endif -} - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_pop_4_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_pop_4_error(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ - int opcode = next_instr->op.code; - STACK_SHRINK(1); - #ifdef LLTRACE - __attribute__((musttail)) - return _TAIL_CALL_pop_3_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace); - #else - __attribute__((musttail)) - return _TAIL_CALL_pop_3_error(frame, stack_pointer, tstate, next_instr, oparg, entry_frame); - #endif + /* END_BASE_INTERPRETER */ } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -341,19 +208,9 @@ _TAIL_CALL_BINARY_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_OP_ADD_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_OP_ADD_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -391,19 +248,9 @@ _TAIL_CALL_BINARY_OP_ADD_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_po } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_OP_ADD_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_OP_ADD_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -440,19 +287,9 @@ _TAIL_CALL_BINARY_OP_ADD_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_poin } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_OP_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_OP_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -489,19 +326,9 @@ _TAIL_CALL_BINARY_OP_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef *stack_ } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -564,19 +391,9 @@ _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(_PyInterpreterFrame *frame, _PyStackRef } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -614,19 +431,9 @@ _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *sta } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_OP_MULTIPLY_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_OP_MULTIPLY_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -663,19 +470,9 @@ _TAIL_CALL_BINARY_OP_MULTIPLY_INT(_PyInterpreterFrame *frame, _PyStackRef *stack } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -713,19 +510,9 @@ _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *sta } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_OP_SUBTRACT_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_OP_SUBTRACT_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -762,19 +549,9 @@ _TAIL_CALL_BINARY_OP_SUBTRACT_INT(_PyInterpreterFrame *frame, _PyStackRef *stack } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -826,19 +603,9 @@ _TAIL_CALL_BINARY_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -887,19 +654,9 @@ _TAIL_CALL_BINARY_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -936,19 +693,9 @@ _TAIL_CALL_BINARY_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_poi } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_SUBSCR_GETITEM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_SUBSCR_GETITEM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -1009,19 +756,9 @@ _TAIL_CALL_BINARY_SUBSCR_GETITEM(_PyInterpreterFrame *frame, _PyStackRef *stack_ } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -1063,19 +800,9 @@ _TAIL_CALL_BINARY_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_SUBSCR_STR_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_SUBSCR_STR_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -1109,19 +836,9 @@ _TAIL_CALL_BINARY_SUBSCR_STR_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_ } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -1155,19 +872,9 @@ _TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(_PyInterpreterFrame *frame, _PyStackRef *stac } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BUILD_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BUILD_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -1189,19 +896,9 @@ _TAIL_CALL_BUILD_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BUILD_MAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BUILD_MAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -1243,19 +940,9 @@ _TAIL_CALL_BUILD_MAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BUILD_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BUILD_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -1301,19 +988,9 @@ _TAIL_CALL_BUILD_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BUILD_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BUILD_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -1345,19 +1022,9 @@ _TAIL_CALL_BUILD_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BUILD_STRING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BUILD_STRING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -1394,19 +1061,9 @@ _TAIL_CALL_BUILD_STRING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BUILD_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_BUILD_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -1428,19 +1085,9 @@ _TAIL_CALL_BUILD_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CACHE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CACHE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CACHE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -1451,19 +1098,9 @@ _TAIL_CALL_CACHE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -1623,19 +1260,9 @@ _TAIL_CALL_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 4; @@ -1735,19 +1362,9 @@ _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(_PyInterpreterFrame *frame, _PyStackRef *st } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 4; @@ -1852,19 +1469,9 @@ _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 4; @@ -1965,19 +1572,9 @@ _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *st } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_BUILTIN_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_BUILTIN_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -2056,19 +1653,9 @@ _TAIL_CALL_CALL_BUILTIN_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_poi } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_BUILTIN_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_BUILTIN_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -2153,19 +1740,9 @@ _TAIL_CALL_CALL_BUILTIN_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_poin } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -2251,19 +1828,9 @@ _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackR } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_BUILTIN_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_BUILTIN_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -2333,19 +1900,9 @@ _TAIL_CALL_CALL_BUILTIN_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -2513,19 +2070,9 @@ _TAIL_CALL_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_point } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_INTRINSIC_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_INTRINSIC_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -2545,19 +2092,9 @@ _TAIL_CALL_CALL_INTRINSIC_1(_PyInterpreterFrame *frame, _PyStackRef *stack_point } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_INTRINSIC_2(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_INTRINSIC_2(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -2584,19 +2121,9 @@ _TAIL_CALL_CALL_INTRINSIC_2(_PyInterpreterFrame *frame, _PyStackRef *stack_point } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_ISINSTANCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_ISINSTANCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -2642,19 +2169,9 @@ _TAIL_CALL_CALL_ISINSTANCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -2813,19 +2330,9 @@ _TAIL_CALL_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_KW_BOUND_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_KW_BOUND_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 4; @@ -2932,19 +2439,9 @@ _TAIL_CALL_CALL_KW_BOUND_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_p } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_KW_NON_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_KW_NON_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -3038,19 +2535,9 @@ _TAIL_CALL_CALL_KW_NON_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_KW_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_KW_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 4; @@ -3136,19 +2623,9 @@ _TAIL_CALL_CALL_KW_PY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -3197,19 +2674,9 @@ _TAIL_CALL_CALL_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -3249,19 +2716,9 @@ _TAIL_CALL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_point } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -3348,19 +2805,9 @@ _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(_PyInterpreterFrame *frame, _PyStackRef * } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -3447,19 +2894,9 @@ _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(_PyInterpreterFrame *frame, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -3533,19 +2970,9 @@ _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(_PyInterpreterFrame *frame, _PyStackRef } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -3622,19 +3049,9 @@ _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(_PyInterpreterFrame *frame, _PyStackRef *sta } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_NON_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_NON_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -3722,19 +3139,9 @@ _TAIL_CALL_CALL_NON_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_po } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_PY_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_PY_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 4; @@ -3817,19 +3224,9 @@ _TAIL_CALL_CALL_PY_EXACT_ARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_poi } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 4; @@ -3909,19 +3306,9 @@ _TAIL_CALL_CALL_PY_GENERAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_STR_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_STR_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -3974,19 +3361,9 @@ _TAIL_CALL_CALL_STR_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_TUPLE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_TUPLE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -4039,19 +3416,9 @@ _TAIL_CALL_CALL_TUPLE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_TYPE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CALL_TYPE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -4081,19 +3448,9 @@ _TAIL_CALL_CALL_TYPE_1(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CHECK_EG_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CHECK_EG_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -4142,19 +3499,9 @@ _TAIL_CALL_CHECK_EG_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CHECK_EXC_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CHECK_EXC_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -4184,19 +3531,9 @@ _TAIL_CALL_CHECK_EXC_MATCH(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CLEANUP_THROW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CLEANUP_THROW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -4240,19 +3577,9 @@ _TAIL_CALL_CLEANUP_THROW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_COMPARE_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_COMPARE_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -4315,19 +3642,9 @@ _TAIL_CALL_COMPARE_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_COMPARE_OP_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_COMPARE_OP_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -4367,19 +3684,9 @@ _TAIL_CALL_COMPARE_OP_FLOAT(_PyInterpreterFrame *frame, _PyStackRef *stack_point } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_COMPARE_OP_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_COMPARE_OP_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -4423,19 +3730,9 @@ _TAIL_CALL_COMPARE_OP_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_COMPARE_OP_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_COMPARE_OP_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -4476,19 +3773,9 @@ _TAIL_CALL_COMPARE_OP_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CONTAINS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CONTAINS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -4536,19 +3823,9 @@ _TAIL_CALL_CONTAINS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CONTAINS_OP_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CONTAINS_OP_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -4578,19 +3855,9 @@ _TAIL_CALL_CONTAINS_OP_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_point } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CONTAINS_OP_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CONTAINS_OP_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -4621,19 +3888,9 @@ _TAIL_CALL_CONTAINS_OP_SET(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CONVERT_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_CONVERT_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -4655,19 +3912,9 @@ _TAIL_CALL_CONVERT_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_COPY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_COPY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_COPY(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -4684,19 +3931,9 @@ _TAIL_CALL_COPY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_COPY_FREE_VARS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_COPY_FREE_VARS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -4716,19 +3953,9 @@ _TAIL_CALL_COPY_FREE_VARS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_DELETE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_DELETE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -4747,19 +3974,9 @@ _TAIL_CALL_DELETE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_DELETE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_DELETE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -4779,19 +3996,9 @@ _TAIL_CALL_DELETE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_DELETE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_DELETE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -4811,19 +4018,9 @@ _TAIL_CALL_DELETE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_DELETE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_DELETE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -4847,19 +4044,9 @@ _TAIL_CALL_DELETE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_DELETE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_DELETE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -4890,19 +4077,9 @@ _TAIL_CALL_DELETE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_DELETE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_DELETE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -4925,19 +4102,9 @@ _TAIL_CALL_DELETE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_DICT_MERGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_DICT_MERGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -4968,19 +4135,9 @@ _TAIL_CALL_DICT_MERGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_DICT_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_DICT_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -5015,19 +4172,9 @@ _TAIL_CALL_DICT_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_END_ASYNC_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_END_ASYNC_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -5060,19 +4207,9 @@ _TAIL_CALL_END_ASYNC_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_END_FOR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -5086,19 +4223,9 @@ _TAIL_CALL_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_END_SEND(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -5118,19 +4245,9 @@ _TAIL_CALL_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_ENTER_EXECUTOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_ENTER_EXECUTOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -5165,19 +4282,9 @@ _TAIL_CALL_ENTER_EXECUTOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_EXIT_INIT_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_EXIT_INIT_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -5199,19 +4306,9 @@ _TAIL_CALL_EXIT_INIT_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_EXTENDED_ARG(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_EXTENDED_ARG(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -5224,19 +4321,9 @@ _TAIL_CALL_EXTENDED_ARG(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_FORMAT_SIMPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_FORMAT_SIMPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -5263,19 +4350,9 @@ _TAIL_CALL_FORMAT_SIMPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_FORMAT_WITH_SPEC(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_FORMAT_WITH_SPEC(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -5299,19 +4376,9 @@ _TAIL_CALL_FORMAT_WITH_SPEC(_PyInterpreterFrame *frame, _PyStackRef *stack_point } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -5377,19 +4444,9 @@ _TAIL_CALL_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_FOR_ITER_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_FOR_ITER_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -5439,19 +4496,9 @@ _TAIL_CALL_FOR_ITER_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_FOR_ITER_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_FOR_ITER_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -5504,19 +4551,9 @@ _TAIL_CALL_FOR_ITER_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_FOR_ITER_RANGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_FOR_ITER_RANGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -5563,19 +4600,9 @@ _TAIL_CALL_FOR_ITER_RANGE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_FOR_ITER_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_FOR_ITER_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -5625,19 +4652,9 @@ _TAIL_CALL_FOR_ITER_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_GET_AITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_GET_AITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -5686,19 +4703,9 @@ _TAIL_CALL_GET_AITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_GET_ANEXT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_GET_ANEXT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -5720,19 +4727,9 @@ _TAIL_CALL_GET_ANEXT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_GET_AWAITABLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_GET_AWAITABLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -5751,19 +4748,9 @@ _TAIL_CALL_GET_AWAITABLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_GET_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_GET_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -5783,19 +4770,9 @@ _TAIL_CALL_GET_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_GET_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_GET_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -5818,19 +4795,9 @@ _TAIL_CALL_GET_LEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_GET_YIELD_FROM_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_GET_YIELD_FROM_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -5875,19 +4842,9 @@ _TAIL_CALL_GET_YIELD_FROM_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_po } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_IMPORT_FROM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_IMPORT_FROM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -5908,19 +4865,9 @@ _TAIL_CALL_IMPORT_FROM(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_IMPORT_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_IMPORT_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -5947,19 +4894,9 @@ _TAIL_CALL_IMPORT_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -6127,19 +5064,9 @@ _TAIL_CALL_INSTRUMENTED_CALL(_PyInterpreterFrame *frame, _PyStackRef *stack_poin } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -6148,19 +5075,9 @@ _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(_PyInterpreterFrame *frame, _PyStackRef } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -6186,19 +5103,9 @@ _TAIL_CALL_INSTRUMENTED_CALL_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_p } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -6225,19 +5132,9 @@ _TAIL_CALL_INSTRUMENTED_END_FOR(_PyInterpreterFrame *frame, _PyStackRef *stack_p } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -6266,19 +5163,9 @@ _TAIL_CALL_INSTRUMENTED_END_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_ } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -6319,19 +5206,9 @@ _TAIL_CALL_INSTRUMENTED_FOR_ITER(_PyInterpreterFrame *frame, _PyStackRef *stack_ } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_INSTRUCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_INSTRUCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -6352,19 +5229,9 @@ _TAIL_CALL_INSTRUMENTED_INSTRUCTION(_PyInterpreterFrame *frame, _PyStackRef *sta } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -6390,19 +5257,9 @@ _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *s } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -6413,19 +5270,9 @@ _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *st } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_LINE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_LINE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -6464,19 +5311,9 @@ _TAIL_CALL_INSTRUMENTED_LINE(_PyInterpreterFrame *frame, _PyStackRef *stack_poin } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -6490,19 +5327,9 @@ _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -6513,19 +5340,9 @@ _TAIL_CALL_INSTRUMENTED_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -6543,19 +5360,9 @@ _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRe } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -6575,19 +5382,9 @@ _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -6605,19 +5402,9 @@ _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStac } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -6635,19 +5422,9 @@ _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -6721,19 +5498,9 @@ _TAIL_CALL_INSTRUMENTED_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_po } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -6780,19 +5547,9 @@ _TAIL_CALL_INSTRUMENTED_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *st } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -6863,19 +5620,9 @@ _TAIL_CALL_INSTRUMENTED_YIELD_VALUE(_PyInterpreterFrame *frame, _PyStackRef *sta } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INTERPRETER_EXIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_INTERPRETER_EXIT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -6896,19 +5643,9 @@ _TAIL_CALL_INTERPRETER_EXIT(_PyInterpreterFrame *frame, _PyStackRef *stack_point } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_IS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_IS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_IS_OP(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -6929,19 +5666,9 @@ _TAIL_CALL_IS_OP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -7001,19 +5728,9 @@ _TAIL_CALL_JUMP_BACKWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -7028,19 +5745,9 @@ _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(_PyInterpreterFrame *frame, _PyStackRef *s } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -7050,19 +5757,9 @@ _TAIL_CALL_JUMP_FORWARD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -7080,19 +5777,9 @@ _TAIL_CALL_LIST_APPEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LIST_EXTEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LIST_EXTEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -7131,19 +5818,9 @@ _TAIL_CALL_LIST_EXTEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 10; @@ -7223,19 +5900,9 @@ _TAIL_CALL_LOAD_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -7272,19 +5939,9 @@ _TAIL_CALL_LOAD_ATTR_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -7327,19 +5984,9 @@ _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(_PyInterpreterFrame *frame, _PyS } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -7377,19 +6024,9 @@ _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(_PyInterpreterFrame *frame, _PyStac } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -7436,19 +6073,9 @@ _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *sta } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -7493,19 +6120,9 @@ _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(_PyInterpreterFrame *frame, _PyStackRef *s } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -7543,19 +6160,9 @@ _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *sta } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -7605,19 +6212,9 @@ _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -7671,19 +6268,9 @@ _TAIL_CALL_LOAD_ATTR_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_point } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -7716,19 +6303,9 @@ _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(_PyInterpreterFrame *frame, _PyStackR } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -7772,19 +6349,9 @@ _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(_PyInterpreterFrame *frame, _PySt } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_PROPERTY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_PROPERTY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -7851,19 +6418,9 @@ _TAIL_CALL_LOAD_ATTR_PROPERTY(_PyInterpreterFrame *frame, _PyStackRef *stack_poi } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -7902,19 +6459,9 @@ _TAIL_CALL_LOAD_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -7968,19 +6515,9 @@ _TAIL_CALL_LOAD_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_po } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_BUILD_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_BUILD_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -8006,19 +6543,9 @@ _TAIL_CALL_LOAD_BUILD_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_point } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_COMMON_CONSTANT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_COMMON_CONSTANT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -8042,19 +6569,9 @@ _TAIL_CALL_LOAD_COMMON_CONSTANT(_PyInterpreterFrame *frame, _PyStackRef *stack_p } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_CONST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_CONST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -8069,19 +6586,9 @@ _TAIL_CALL_LOAD_CONST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_CONST_IMMORTAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_CONST_IMMORTAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -8098,19 +6605,9 @@ _TAIL_CALL_LOAD_CONST_IMMORTAL(_PyInterpreterFrame *frame, _PyStackRef *stack_po } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -8132,19 +6629,9 @@ _TAIL_CALL_LOAD_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -8159,19 +6646,9 @@ _TAIL_CALL_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_FAST_AND_CLEAR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_FAST_AND_CLEAR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -8187,19 +6664,9 @@ _TAIL_CALL_LOAD_FAST_AND_CLEAR(_PyInterpreterFrame *frame, _PyStackRef *stack_po } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_FAST_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_FAST_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -8223,19 +6690,9 @@ _TAIL_CALL_LOAD_FAST_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -8254,19 +6711,9 @@ _TAIL_CALL_LOAD_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_po } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -8303,19 +6750,9 @@ _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stac } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -8387,19 +6824,9 @@ _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(_PyInterpreterFrame *frame, _PyStackRef *st } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 5; @@ -8446,19 +6873,9 @@ _TAIL_CALL_LOAD_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_GLOBAL_BUILTIN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_GLOBAL_BUILTIN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 5; @@ -8511,19 +6928,9 @@ _TAIL_CALL_LOAD_GLOBAL_BUILTIN(_PyInterpreterFrame *frame, _PyStackRef *stack_po } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_GLOBAL_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_GLOBAL_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 5; @@ -8568,19 +6975,9 @@ _TAIL_CALL_LOAD_GLOBAL_MODULE(_PyInterpreterFrame *frame, _PyStackRef *stack_poi } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_LOCALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_LOCALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -8602,19 +6999,9 @@ _TAIL_CALL_LOAD_LOCALS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -8633,19 +7020,9 @@ _TAIL_CALL_LOAD_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_SMALL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_SMALL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -8661,19 +7038,9 @@ _TAIL_CALL_LOAD_SMALL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_SPECIAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_SPECIAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -8712,19 +7079,9 @@ _TAIL_CALL_LOAD_SPECIAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -8825,19 +7182,9 @@ _TAIL_CALL_LOAD_SUPER_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -8874,19 +7221,9 @@ _TAIL_CALL_LOAD_SUPER_ATTR_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_p } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -8936,19 +7273,9 @@ _TAIL_CALL_LOAD_SUPER_ATTR_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_MAKE_CELL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_MAKE_CELL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -8965,19 +7292,9 @@ _TAIL_CALL_MAKE_CELL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_MAKE_FUNCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_MAKE_FUNCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -9000,19 +7317,9 @@ _TAIL_CALL_MAKE_FUNCTION(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_MAP_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_MAP_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -9041,19 +7348,9 @@ _TAIL_CALL_MAP_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_MATCH_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_MATCH_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -9093,19 +7390,9 @@ _TAIL_CALL_MATCH_CLASS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_MATCH_KEYS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_MATCH_KEYS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -9129,19 +7416,9 @@ _TAIL_CALL_MATCH_KEYS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_MATCH_MAPPING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_MATCH_MAPPING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -9158,19 +7435,9 @@ _TAIL_CALL_MATCH_MAPPING(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_MATCH_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_MATCH_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -9187,19 +7454,9 @@ _TAIL_CALL_MATCH_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_NOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_NOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_NOP(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -9208,19 +7465,9 @@ _TAIL_CALL_NOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -9229,19 +7476,9 @@ _TAIL_CALL_NOT_TAKEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_POP_EXCEPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_POP_EXCEPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -9260,19 +7497,9 @@ _TAIL_CALL_POP_EXCEPT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -9291,19 +7518,9 @@ _TAIL_CALL_POP_JUMP_IF_FALSE(_PyInterpreterFrame *frame, _PyStackRef *stack_poin } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -9338,19 +7555,9 @@ _TAIL_CALL_POP_JUMP_IF_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_point } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -9385,19 +7592,9 @@ _TAIL_CALL_POP_JUMP_IF_NOT_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_p } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -9416,19 +7613,9 @@ _TAIL_CALL_POP_JUMP_IF_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_point } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_POP_TOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_POP_TOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -9442,19 +7629,9 @@ _TAIL_CALL_POP_TOP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_PUSH_EXC_INFO(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_PUSH_EXC_INFO(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -9481,19 +7658,9 @@ _TAIL_CALL_PUSH_EXC_INFO(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_PUSH_NULL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_PUSH_NULL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -9507,19 +7674,9 @@ _TAIL_CALL_PUSH_NULL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_RAISE_VARARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_RAISE_VARARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -9546,19 +7703,9 @@ _TAIL_CALL_RAISE_VARARGS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_RERAISE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_RERAISE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_RERAISE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -9604,19 +7751,9 @@ _TAIL_CALL_RERAISE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_RESERVED(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_RESERVED(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_RESERVED(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -9627,19 +7764,9 @@ _TAIL_CALL_RESERVED(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_RESUME(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -9711,19 +7838,9 @@ _TAIL_CALL_RESUME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_RESUME_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_RESUME_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -9745,19 +7862,9 @@ _TAIL_CALL_RESUME_CHECK(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_RETURN_GENERATOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_RETURN_GENERATOR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -9792,19 +7899,9 @@ _TAIL_CALL_RETURN_GENERATOR(_PyInterpreterFrame *frame, _PyStackRef *stack_point } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -9836,19 +7933,9 @@ _TAIL_CALL_RETURN_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_SEND(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -9940,19 +8027,9 @@ _TAIL_CALL_SEND(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_SEND_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_SEND_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -10006,19 +8083,9 @@ _TAIL_CALL_SEND_GEN(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_SETUP_ANNOTATIONS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_SETUP_ANNOTATIONS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -10055,19 +8122,9 @@ _TAIL_CALL_SETUP_ANNOTATIONS(_PyInterpreterFrame *frame, _PyStackRef *stack_poin } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_SET_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_SET_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -10088,19 +8145,9 @@ _TAIL_CALL_SET_ADD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -10126,19 +8173,9 @@ _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(_PyInterpreterFrame *frame, _PyStackRef *stack } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_SET_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_SET_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -10159,19 +8196,9 @@ _TAIL_CALL_SET_UPDATE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 5; @@ -10218,19 +8245,9 @@ _TAIL_CALL_STORE_ATTR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 5; @@ -10288,19 +8305,9 @@ _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(_PyInterpreterFrame *frame, _PyStackRef *st } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 5; @@ -10337,19 +8344,9 @@ _TAIL_CALL_STORE_ATTR_SLOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 5; @@ -10415,19 +8412,9 @@ _TAIL_CALL_STORE_ATTR_WITH_HINT(_PyInterpreterFrame *frame, _PyStackRef *stack_p } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -10444,19 +8431,9 @@ _TAIL_CALL_STORE_DEREF(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -10470,19 +8447,9 @@ _TAIL_CALL_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -10499,19 +8466,9 @@ _TAIL_CALL_STORE_FAST_LOAD_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_p } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_FAST_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_FAST_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -10530,19 +8487,9 @@ _TAIL_CALL_STORE_FAST_STORE_FAST(_PyInterpreterFrame *frame, _PyStackRef *stack_ } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -10561,19 +8508,9 @@ _TAIL_CALL_STORE_GLOBAL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -10609,19 +8546,9 @@ _TAIL_CALL_STORE_NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -10671,19 +8598,9 @@ _TAIL_CALL_STORE_SLICE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -10730,19 +8647,9 @@ _TAIL_CALL_STORE_SUBSCR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -10771,19 +8678,9 @@ _TAIL_CALL_STORE_SUBSCR_DICT(_PyInterpreterFrame *frame, _PyStackRef *stack_poin } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_STORE_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -10823,19 +8720,9 @@ _TAIL_CALL_STORE_SUBSCR_LIST_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_ } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_SWAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_SWAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_SWAP(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -10855,19 +8742,9 @@ _TAIL_CALL_SWAP(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_TO_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_TO_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -10909,19 +8786,9 @@ _TAIL_CALL_TO_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; next_instr += 4; @@ -10950,19 +8817,9 @@ _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(_PyInterpreterFrame *frame, _PyStackRef *stack_po } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_TO_BOOL_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_TO_BOOL_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -10978,19 +8835,9 @@ _TAIL_CALL_TO_BOOL_BOOL(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_TO_BOOL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_TO_BOOL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -11017,19 +8864,9 @@ _TAIL_CALL_TO_BOOL_INT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_TO_BOOL_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_TO_BOOL_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -11050,19 +8887,9 @@ _TAIL_CALL_TO_BOOL_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_TO_BOOL_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_TO_BOOL_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -11082,19 +8909,9 @@ _TAIL_CALL_TO_BOOL_NONE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_TO_BOOL_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_TO_BOOL_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 4; @@ -11122,19 +8939,9 @@ _TAIL_CALL_TO_BOOL_STR(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_UNARY_INVERT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_UNARY_INVERT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -11153,19 +8960,9 @@ _TAIL_CALL_UNARY_INVERT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_UNARY_NEGATIVE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_UNARY_NEGATIVE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -11184,19 +8981,9 @@ _TAIL_CALL_UNARY_NEGATIVE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_UNARY_NOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_UNARY_NOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -11212,19 +8999,9 @@ _TAIL_CALL_UNARY_NOT(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_UNPACK_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_UNPACK_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -11245,19 +9022,9 @@ _TAIL_CALL_UNPACK_EX(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_UNPACK_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_UNPACK_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -11302,19 +9069,9 @@ _TAIL_CALL_UNPACK_SEQUENCE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointe } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_UNPACK_SEQUENCE_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_UNPACK_SEQUENCE_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -11345,19 +9102,9 @@ _TAIL_CALL_UNPACK_SEQUENCE_LIST(_PyInterpreterFrame *frame, _PyStackRef *stack_p } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -11383,19 +9130,9 @@ _TAIL_CALL_UNPACK_SEQUENCE_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_ } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 2; @@ -11422,19 +9159,9 @@ _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(_PyInterpreterFrame *frame, _PyStackRef *st } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_WITH_EXCEPT_START(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_WITH_EXCEPT_START(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; @@ -11486,19 +9213,9 @@ _TAIL_CALL_WITH_EXCEPT_START(_PyInterpreterFrame *frame, _PyStackRef *stack_poin } } - -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_YIELD_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_YIELD_VALUE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -{ +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; + (void)(opcode); { frame->instr_ptr = next_instr; next_instr += 1; diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index 59fe06735dac28..3b5761a3238314 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -4,6 +4,7 @@ """ import argparse +import re from typing import TextIO @@ -26,37 +27,44 @@ DEFAULT_INPUT = ROOT / "Python/generated_tail_call_handlers.c.h" DEFAULT_OUTPUT = ROOT / "Python/generated_tail_call_handlers.c.h" +DEFAULT_CEVAL_INPUT = ROOT / "Python/ceval.c" FOOTER = "#undef TIER_ONE\n" -def function_proto(name: str) -> str: - return f""" -#ifdef LLTRACE -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_{name}(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame, int lltrace) -#else - -__attribute__((preserve_none)) static PyObject * -_TAIL_CALL_{name}(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, - PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame* entry_frame) -#endif -""" +TARGET_LABEL = "TAIL_CALL_TARGET" + +def generate_label_handlers(outfile: TextIO): + out = CWriter(outfile, 0, False) + with open(DEFAULT_CEVAL_INPUT, "r") as fp: + str_in = fp.read() + # https://stackoverflow.com/questions/8303488/regex-to-match-any-character-including-new-lines + eval_framedefault = re.findall("_PyEval_EvalFrameDefault\(.*\)\n({[\s\S]*\/\* END_BASE_INTERPRETER \*\/)", str_in)[0] + function_protos = re.findall(f"{TARGET_LABEL}\((\w+)\):", eval_framedefault) + for proto in function_protos: + out.emit(f"{function_proto(proto)};\n") + lines = iter(eval_framedefault[eval_framedefault.find(TARGET_LABEL):].split("\n")) + next(lines) + for i in range(len(function_protos)): + curr_proto = function_protos[i] + fallthrough_proto = function_protos[i + 1] if i + 1 < len(function_protos) else None + out.emit(function_proto(curr_proto)) + out.emit("\n") + out.emit("{\n") + for line in lines: + if TARGET_LABEL in line: + break + if label := re.findall("goto (\w+);", line): + out.emit(f"CEVAL_GOTO({label[0]});\n") + else: + out.emit(line) + out.emit("\n") + if fallthrough_proto: + out.emit(f"CEVAL_GOTO({fallthrough_proto});\n") + out.emit("}\n") -def error_handler(out: CWriter, body: str, next: str, emit_tail_call: bool = True): - out.emit("{\n") - out.emit("int opcode = next_instr->op.code;\n") - out.emit(body) - if emit_tail_call: - out.emit("#ifdef LLTRACE\n") - out.emit("__attribute__((musttail))\n") - out.emit(f"return _TAIL_CALL_{next}(frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace);\n") - out.emit("#else\n") - out.emit("__attribute__((musttail))\n") - out.emit(f"return _TAIL_CALL_{next}(frame, stack_pointer, tstate, next_instr, oparg, entry_frame);\n") - out.emit("#endif\n") - out.emit("}\n") +def function_proto(name: str) -> str: + return f"Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_{name}(TAIL_CALL_PARAMS)" def generate_tier1( filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool @@ -71,144 +79,12 @@ def generate_tier1( """ ) out = CWriter(outfile, 0, lines) + out.emit("static inline PyObject *_TAIL_CALL_shim(TAIL_CALL_PARAMS);\n") out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256];\n"); # Emit error handlers - out.emit(function_proto("error")) - out.emit(";\n") - - out.emit(function_proto("resume_with_error")) - body = """ - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - """ - next = "error" - error_handler(out, body, next) - - out.emit(function_proto("exit_unwind")) - body = """ - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - """ - next = "resume_with_error" - error_handler(out, body, next) - - out.emit(function_proto("exception_unwind")) - body = """ - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - CEVAL_GOTO(exit_unwind); - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - CEVAL_GOTO(exception_unwind); - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - CEVAL_GOTO(exception_unwind); - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } -""" - next = "no target" - error_handler(out, body, next, emit_tail_call=False) - - out.emit(function_proto("error")) - body = """ - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -""" - next = "exception_unwind" - error_handler(out, body, next) - - out.emit(function_proto("pop_1_error")) - body = "STACK_SHRINK(1);\n" - next = "error" - error_handler(out, body, next) - - out.emit(function_proto("pop_2_error")) - body = "STACK_SHRINK(1);\n" - next = "pop_1_error" - error_handler(out, body, next) - - out.emit(function_proto("pop_3_error")) - body = "STACK_SHRINK(1);\n" - next = "pop_2_error" - error_handler(out, body, next) - - out.emit(function_proto("pop_4_error")) - body = "STACK_SHRINK(1);\n" - next = "pop_3_error" - error_handler(out, body, next) - + generate_label_handlers(outfile) emitter = Emitter(out) out.emit("\n") @@ -217,6 +93,8 @@ def generate_tier1( out.emit(function_proto(name)) out.emit("{\n") out.emit("int opcode = next_instr->op.code;\n") + # Some instructions don't use opcode. + out.emit(f"(void)(opcode);\n") out.emit("{\n") write_single_inst(out, emitter, name, inst) if not inst.parts[-1].properties.always_exits: From 637589e927598bbd3bcfe6c5bcaba8d4e68871c8 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 10 Jan 2025 08:31:47 +0800 Subject: [PATCH 019/110] Revert "Make deopt more efficient" This reverts commit 982c51d05ad6b6bdc9d2f9ce12542f91e2810ffe. --- Python/bytecodes.c | 8 +- Python/ceval_macros.h | 12 +- Python/generated_cases.c.h | 450 ++++++++++----------- Python/generated_tail_call_handlers.c.h | 450 ++++++++++----------- Tools/cases_generator/generators_common.py | 1 - 5 files changed, 460 insertions(+), 461 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6dffc016dcb252..97d9430a96749b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -45,7 +45,7 @@ #include "ceval_macros.h" /* Flow control macros */ -#define GO_TO_INSTRUCTION(instname, SIZE) ((void)0) +#define GO_TO_INSTRUCTION(instname) ((void)0) #define inst(name, ...) case name: #define op(name, ...) /* NAME is ignored */ @@ -1942,7 +1942,7 @@ dummy_func( // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR, INSTRUCTION_SIZE); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } family(LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR) = { @@ -4255,7 +4255,7 @@ dummy_func( frame, this_instr, function, arg); ERROR_IF(err, error); PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(CALL_KW, INSTRUCTION_SIZE); + GO_TO_INSTRUCTION(CALL_KW); } op(_MAYBE_EXPAND_METHOD_KW, (callable[1], self_or_null[1], args[oparg], kwnames_in -- func[1], maybe_self[1], args[oparg], kwnames_out)) { @@ -4488,7 +4488,7 @@ dummy_func( _CHECK_PERIODIC; inst(INSTRUMENTED_CALL_FUNCTION_EX, ( -- )) { - GO_TO_INSTRUCTION(CALL_FUNCTION_EX, INSTRUCTION_SIZE); + GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } op(_MAKE_CALLARGS_A_TUPLE, (func, unused, callargs, kwargs_in if (oparg & 1) -- func, unused, tuple, kwargs_out if (oparg & 1))) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index fb07c87977e46a..27aeb98fb2bf00 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -329,14 +329,14 @@ GETITEM(PyObject *v, Py_ssize_t i) { PyStackRef_XCLOSE(tmp); } while (0) #ifdef Py_TAIL_CALL_INTERP #ifdef LLTRACE -#define GO_TO_INSTRUCTION(op, SIZE) do { \ +#define GO_TO_INSTRUCTION(op) do { \ Py_MUSTTAIL \ - return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - SIZE, oparg, entry_frame, lltrace); \ + return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - _PyOpcode_Caches[_PyOpcode_Deopt[op]], oparg, entry_frame, lltrace); \ } while (0) #else -#define GO_TO_INSTRUCTION(op, SIZE) do { \ +#define GO_TO_INSTRUCTION(op) do { \ Py_MUSTTAIL \ - return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - SIZE, oparg, entry_frame); \ + return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - _PyOpcode_Caches[_PyOpcode_Deopt[op]], oparg, entry_frame); \ } while (0) #endif #else @@ -357,12 +357,12 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define UPDATE_MISS_STATS(INSTNAME) ((void)0) #endif -#define DEOPT_IF(COND, INSTNAME, SIZE) \ +#define DEOPT_IF(COND, INSTNAME) \ if ((COND)) { \ /* This is only a single jump on release builds! */ \ UPDATE_MISS_STATS((INSTNAME)); \ /* assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); */ \ - GO_TO_INSTRUCTION(INSTNAME, SIZE); \ + GO_TO_INSTRUCTION(INSTNAME); \ } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 9dc2c98418a4c3..439f1424f3465e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -72,8 +72,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_ADD_FLOAT @@ -108,8 +108,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_ADD_INT @@ -143,8 +143,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_ADD_UNICODE @@ -177,8 +177,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_INPLACE_ADD_UNICODE @@ -193,7 +193,7 @@ next_oparg = CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); - DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -239,8 +239,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_MULTIPLY_FLOAT @@ -275,8 +275,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_MULTIPLY_INT @@ -310,8 +310,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_SUBTRACT_FLOAT @@ -346,8 +346,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_SUBTRACT_INT @@ -477,7 +477,7 @@ dict_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -511,22 +511,22 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); } // _BINARY_SUBSCR_CHECK_FUNC { container = stack_pointer[-2]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR); assert(PyFunction_Check(getitem_o)); uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR); PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); } @@ -571,19 +571,19 @@ list_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; #ifdef Py_GIL_DISABLED _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); stack_pointer = _PyFrame_GetStackPointer(frame); - DEOPT_IF(res_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(res_o == NULL, BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); #else - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); @@ -611,14 +611,14 @@ str_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); @@ -643,12 +643,12 @@ tuple_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); @@ -1033,7 +1033,7 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_AND_ALLOCATE_OBJECT { @@ -1044,17 +1044,17 @@ self = &stack_pointer[-1 - oparg]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL); assert(tp->tp_new == PyBaseObject_Type.tp_new); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); assert(tp->tp_alloc == PyType_GenericAlloc); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *self_o = PyType_GenericAlloc(tp, 0); @@ -1131,14 +1131,14 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS { null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL); } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { @@ -1157,9 +1157,9 @@ callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(func->func_version != func_version, CALL); } // _CHECK_FUNCTION_EXACT_ARGS { @@ -1168,15 +1168,15 @@ assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS { @@ -1234,7 +1234,7 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_METHOD_VERSION { @@ -1242,11 +1242,11 @@ callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL); PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyFunction_Check(func), CALL); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); } // _EXPAND_METHOD { @@ -1340,9 +1340,9 @@ args--; total_args++; } - DEOPT_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tp->tp_vectorcall == NULL, CALL); STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { @@ -1418,8 +1418,8 @@ args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ @@ -1501,8 +1501,8 @@ args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1585,11 +1585,11 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; @@ -1860,9 +1860,9 @@ args--; total_args++; } - DEOPT_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(total_args != 2, CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL); STAT_INC(CALL, hit); _PyStackRef cls_stackref = args[1]; _PyStackRef inst_stackref = args[0]; @@ -2056,7 +2056,7 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(tstate->interp->eval_frame, CALL_KW); } // _CHECK_METHOD_VERSION_KW { @@ -2064,11 +2064,11 @@ callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW); PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(!PyFunction_Check(func), CALL_KW); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW); } // _EXPAND_METHOD_KW { @@ -2161,8 +2161,8 @@ { callable = &stack_pointer[-3 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(PyFunction_Check(callable_o), CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW); } // _CALL_KW_NON_PY { @@ -2250,16 +2250,16 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(tstate->interp->eval_frame, CALL_KW); } // _CHECK_FUNCTION_VERSION_KW { callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(func->func_version != func_version, CALL_KW); } // _PY_FRAME_KW { @@ -2342,9 +2342,9 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(total_args != 1, CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.len, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != interp->callable_cache.len, CALL); STAT_INC(CALL, hit); _PyStackRef arg_stackref = args[0]; PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); @@ -2385,10 +2385,10 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL); assert(self_o != NULL); - DEOPT_IF(!PyList_Check(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(!LOCK_OBJECT(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyList_Check(self_o), CALL); + DEOPT_IF(!LOCK_OBJECT(self_o), CALL); STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); @@ -2430,11 +2430,11 @@ } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(args, total_args, args_o); @@ -2514,12 +2514,12 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); PyTypeObject *d_type = method->d_common.d_type; PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(args, total_args, args_o); @@ -2599,16 +2599,16 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(total_args != 1, CALL); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); PyMethodDef *meth = method->d_method; _PyStackRef self_stackref = args[0]; PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -2671,16 +2671,16 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(meth->ml_flags != METH_O, CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); _PyStackRef arg_stackref = args[1]; _PyStackRef self_stackref = args[0]; DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + method->d_common.d_type), CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -2738,8 +2738,8 @@ { callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(PyFunction_Check(callable_o), CALL); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL); } // _CALL_NON_PY_GENERAL { @@ -2821,16 +2821,16 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(func->func_version != func_version, CALL); } // _CHECK_FUNCTION_EXACT_ARGS { @@ -2839,15 +2839,15 @@ assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS { @@ -2902,16 +2902,16 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(func->func_version != func_version, CALL); } // _PY_FRAME_GENERAL { @@ -2987,8 +2987,8 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Str(arg_o); @@ -3038,8 +3038,8 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PySequence_Tuple(arg_o); @@ -3087,8 +3087,8 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL); STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); PyStackRef_CLOSE(arg); @@ -3290,8 +3290,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_FLOAT @@ -3328,16 +3328,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_INT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && _PyLong_DigitCount((PyLongObject *)right_o) <= 1); @@ -3370,8 +3370,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_STR @@ -3454,7 +3454,7 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); + DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); @@ -3482,7 +3482,7 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); + DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3985,14 +3985,14 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); } // _FOR_ITER_GEN_FRAME { iter = stack_pointer[-1]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); STAT_INC(FOR_ITER, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, PyStackRef_None); @@ -4033,7 +4033,7 @@ // _ITER_CHECK_LIST { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER); } // _ITER_JUMP_LIST { @@ -4085,7 +4085,7 @@ { iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); } // _ITER_JUMP_RANGE { @@ -4129,7 +4129,7 @@ // _ITER_CHECK_TUPLE { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER); } // _ITER_JUMP_TUPLE { @@ -4546,7 +4546,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - GO_TO_INSTRUCTION(CALL_FUNCTION_EX, 1 ); + GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } TARGET(INSTRUMENTED_CALL_KW) { @@ -4570,7 +4570,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (err) CEVAL_GOTO(error); PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(CALL_KW, 4 ); + GO_TO_INSTRUCTION(CALL_KW); } TARGET(INSTRUMENTED_END_FOR) { @@ -4762,7 +4762,7 @@ // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR, 2 ); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } TARGET(INSTRUMENTED_NOT_TAKEN) { @@ -5288,9 +5288,9 @@ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS @@ -5323,16 +5323,16 @@ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); } // _GUARD_TYPE_VERSION { uint32_t type_version = read_u32(&this_instr[4].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _LOAD_ATTR_CLASS { @@ -5363,17 +5363,17 @@ PyObject *getattribute = read_obj(&this_instr[6].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR); PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( @@ -5401,14 +5401,14 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _CHECK_MANAGED_OBJECT_HAS_VALUES { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { @@ -5416,7 +5416,7 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = *value_ptr; - DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr_o); null = PyStackRef_NULL; @@ -5446,7 +5446,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _CHECK_ATTR_METHOD_LAZY_DICT { @@ -5454,7 +5454,7 @@ char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = *(PyObject **)ptr; /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(dict != NULL, LOAD_ATTR); } /* Skip 1 cache entry */ // _LOAD_ATTR_METHOD_LAZY_DICT @@ -5489,7 +5489,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_NO_DICT @@ -5525,20 +5525,20 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&this_instr[4].cache); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); } // _LOAD_ATTR_METHOD_WITH_VALUES { @@ -5573,11 +5573,11 @@ owner = stack_pointer[-1]; uint32_t dict_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR); mod_keys = keys; } // _LOAD_ATTR_MODULE_FROM_KEYS @@ -5588,11 +5588,11 @@ PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); // Clear mod_keys from stack in case we need to deopt - DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR); #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); if (!increfed) { - DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR); } #else Py_INCREF(attr_o); @@ -5624,7 +5624,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT @@ -5655,20 +5655,20 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&this_instr[4].cache); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { @@ -5693,7 +5693,7 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); } // _GUARD_TYPE_VERSION { @@ -5701,7 +5701,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_PROPERTY_FRAME @@ -5711,10 +5711,10 @@ assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; PyCodeObject *code = (PyCodeObject *)f->func_code; - DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - DEOPT_IF(code->co_argcount != 1, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR); + DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR); + DEOPT_IF(code->co_argcount != 1, LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); new_frame->localsplus[0] = owner; @@ -5763,7 +5763,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _LOAD_ATTR_SLOT { @@ -5771,7 +5771,7 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); char *addr = (char *)owner_o + index; PyObject *attr_o = *(PyObject **)addr; - DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; attr = PyStackRef_FromPyObjectNew(attr_o); @@ -5800,14 +5800,14 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _CHECK_ATTR_WITH_HINT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(dict == NULL, LOAD_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); } // _LOAD_ATTR_WITH_HINT @@ -5816,13 +5816,13 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject *attr_o; PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), LOAD_ATTR); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(ep->me_key != name, LOAD_ATTR); attr_o = ep->me_value; - DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr_o); attr = PyStackRef_FromPyObjectSteal(attr_o); @@ -6160,18 +6160,18 @@ { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(keys)); } // _GUARD_BUILTINS_VERSION_PUSH_KEYS { uint16_t version = read_u16(&this_instr[3].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); builtins_keys = keys; assert(DK_IS_UNICODE(builtins_keys)); } @@ -6180,10 +6180,10 @@ uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + DEOPT_IF(!increfed, LOAD_GLOBAL); #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -6211,9 +6211,9 @@ { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); globals_keys = keys; assert(DK_IS_UNICODE(globals_keys)); } @@ -6223,10 +6223,10 @@ uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + DEOPT_IF(!increfed, LOAD_GLOBAL); #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -6445,8 +6445,8 @@ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6481,8 +6481,8 @@ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -6998,16 +6998,16 @@ INSTRUCTION_STATS(RESUME_CHECK); static_assert(0 == 0, "incorrect cache size"); #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME, 0); + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME, 0); + DEOPT_IF(eval_breaker != version, RESUME); #ifdef Py_GIL_DISABLED DEOPT_IF(frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME, 0); + ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME); #endif DISPATCH(); } @@ -7177,15 +7177,15 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, SEND, INLINE_CACHE_ENTRIES_SEND); + DEOPT_IF(tstate->interp->eval_frame, SEND); } // _SEND_GEN_FRAME { v = stack_pointer[-1]; receiver = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND, INLINE_CACHE_ENTRIES_SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND, INLINE_CACHE_ENTRIES_SEND); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); STAT_INC(SEND, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, v); @@ -7373,11 +7373,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); PyTypeObject *tp = Py_TYPE(owner_o); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(true, STORE_ATTR); } } // _GUARD_DORV_NO_DICT @@ -7388,7 +7388,7 @@ if (_PyObject_GetManagedDict(owner_o) || !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(true, STORE_ATTR); } } // _STORE_ATTR_INSTANCE_VALUE @@ -7429,14 +7429,14 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); } // _STORE_ATTR_SLOT { value = stack_pointer[-2]; uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; @@ -7464,7 +7464,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); } // _STORE_ATTR_WITH_HINT { @@ -7473,12 +7473,12 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); - DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(dict == NULL, STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR); #ifdef Py_GIL_DISABLED if (dict != _PyObject_GetManagedDict(owner_o)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(true, STORE_ATTR); } #endif assert(PyDict_CheckExact((PyObject *)dict)); @@ -7486,17 +7486,17 @@ if (hint >= (size_t)dict->ma_keys->dk_nentries || !DK_IS_UNICODE(dict->ma_keys)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(true, STORE_ATTR); } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(true, STORE_ATTR); } PyObject *old_value = ep->me_value; if (old_value == NULL) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(true, STORE_ATTR); } _PyFrame_SetStackPointer(frame, stack_pointer); _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); @@ -7730,7 +7730,7 @@ dict_st = stack_pointer[-2]; value = stack_pointer[-3]; PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, @@ -7758,16 +7758,16 @@ value = stack_pointer[-3]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR); // Ensure index < len(list) if (index >= PyList_GET_SIZE(list)) { UNLOCK_OBJECT(list); - DEOPT_IF(true, STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + DEOPT_IF(true, STORE_SUBSCR); } STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); @@ -7855,7 +7855,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL); } // _REPLACE_WITH_TRUE { @@ -7876,7 +7876,7 @@ /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; - DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL); STAT_INC(TO_BOOL, hit); DISPATCH(); } @@ -7892,7 +7892,7 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value_o)) { assert(_Py_IsImmortal(value_o)); @@ -7917,7 +7917,7 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL); STAT_INC(TO_BOOL, hit); res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; PyStackRef_CLOSE(value); @@ -7936,7 +7936,7 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL); STAT_INC(TO_BOOL, hit); res = PyStackRef_False; stack_pointer[-1] = res; @@ -7954,7 +7954,7 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL); STAT_INC(TO_BOOL, hit); if (value_o == &_Py_STR(empty)) { assert(_Py_IsImmortal(value_o)); @@ -8090,11 +8090,11 @@ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); - DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE); if (PyList_GET_SIZE(seq_o) != oparg) { UNLOCK_OBJECT(seq_o); - DEOPT_IF(true, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(true, UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq_o); @@ -8119,8 +8119,8 @@ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { @@ -8144,8 +8144,8 @@ seq = stack_pointer[-1]; assert(oparg == 2); PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index b2a95865d22814..588fc5ee439bd9 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -225,8 +225,8 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_P left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_ADD_FLOAT @@ -265,8 +265,8 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PAR left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_ADD_INT @@ -304,8 +304,8 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_ADD_UNICODE @@ -342,8 +342,8 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(T left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_INPLACE_ADD_UNICODE @@ -358,7 +358,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(T next_oparg = CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); - DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -408,8 +408,8 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_C left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_MULTIPLY_FLOAT @@ -448,8 +448,8 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CAL left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_MULTIPLY_INT @@ -487,8 +487,8 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_C left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_SUBTRACT_FLOAT @@ -527,8 +527,8 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CAL left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_SUBTRACT_INT @@ -670,7 +670,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PA dict_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -708,22 +708,22 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); } // _BINARY_SUBSCR_CHECK_FUNC { container = stack_pointer[-2]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR); assert(PyFunction_Check(getitem_o)); uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR); PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); } @@ -772,19 +772,19 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CAL list_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; #ifdef Py_GIL_DISABLED _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); stack_pointer = _PyFrame_GetStackPointer(frame); - DEOPT_IF(res_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(res_o == NULL, BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); #else - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); @@ -816,14 +816,14 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL str_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); @@ -852,12 +852,12 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CA tuple_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); @@ -1278,7 +1278,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_AND_ALLOCATE_OBJECT { @@ -1289,17 +1289,17 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_ self = &stack_pointer[-1 - oparg]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL); assert(tp->tp_new == PyBaseObject_Type.tp_new); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); assert(tp->tp_alloc == PyType_GenericAlloc); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *self_o = PyType_GenericAlloc(tp, 0); @@ -1380,14 +1380,14 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TA /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS { null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL); } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { @@ -1406,9 +1406,9 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TA callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(func->func_version != func_version, CALL); } // _CHECK_FUNCTION_EXACT_ARGS { @@ -1417,15 +1417,15 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TA assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS { @@ -1487,7 +1487,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_METHOD_VERSION { @@ -1495,11 +1495,11 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_ callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL); PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyFunction_Check(func), CALL); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); } // _EXPAND_METHOD { @@ -1597,9 +1597,9 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PA args--; total_args++; } - DEOPT_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tp->tp_vectorcall == NULL, CALL); STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { @@ -1679,8 +1679,8 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PAR args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ @@ -1766,8 +1766,8 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1854,11 +1854,11 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS args--; total_args++; } - DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; @@ -2145,9 +2145,9 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAM args--; total_args++; } - DEOPT_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(total_args != 2, CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL); STAT_INC(CALL, hit); _PyStackRef cls_stackref = args[1]; _PyStackRef inst_stackref = args[0]; @@ -2349,7 +2349,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(tstate->interp->eval_frame, CALL_KW); } // _CHECK_METHOD_VERSION_KW { @@ -2357,11 +2357,11 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_ callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW); PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(!PyFunction_Check(func), CALL_KW); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW); } // _EXPAND_METHOD_KW { @@ -2458,8 +2458,8 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS { callable = &stack_pointer[-3 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(PyFunction_Check(callable_o), CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW); } // _CALL_KW_NON_PY { @@ -2551,16 +2551,16 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(tstate->interp->eval_frame, CALL_KW); } // _CHECK_FUNCTION_VERSION_KW { callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(func->func_version != func_version, CALL_KW); } // _PY_FRAME_KW { @@ -2647,9 +2647,9 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(total_args != 1, CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.len, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != interp->callable_cache.len, CALL); STAT_INC(CALL, hit); _PyStackRef arg_stackref = args[0]; PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); @@ -2694,10 +2694,10 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARA PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL); assert(self_o != NULL); - DEOPT_IF(!PyList_Check(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(!LOCK_OBJECT(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyList_Check(self_o), CALL); + DEOPT_IF(!LOCK_OBJECT(self_o), CALL); STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); @@ -2743,11 +2743,11 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAI } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(args, total_args, args_o); @@ -2831,12 +2831,12 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WIT total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); PyTypeObject *d_type = method->d_common.d_type; PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(args, total_args, args_o); @@ -2920,16 +2920,16 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(T args--; total_args++; } - DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(total_args != 1, CALL); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); PyMethodDef *meth = method->d_method; _PyStackRef self_stackref = args[0]; PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -2996,16 +2996,16 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_C total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(meth->ml_flags != METH_O, CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); _PyStackRef arg_stackref = args[1]; _PyStackRef self_stackref = args[0]; DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + method->d_common.d_type), CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -3067,8 +3067,8 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_P { callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(PyFunction_Check(callable_o), CALL); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL); } // _CALL_NON_PY_GENERAL { @@ -3154,16 +3154,16 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PA /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(func->func_version != func_version, CALL); } // _CHECK_FUNCTION_EXACT_ARGS { @@ -3172,15 +3172,15 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PA assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS { @@ -3239,16 +3239,16 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAM /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(func->func_version != func_version, CALL); } // _PY_FRAME_GENERAL { @@ -3328,8 +3328,8 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS){ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Str(arg_o); @@ -3383,8 +3383,8 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PySequence_Tuple(arg_o); @@ -3436,8 +3436,8 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS){ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL); STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); PyStackRef_CLOSE(arg); @@ -3659,8 +3659,8 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARA left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_FLOAT @@ -3701,16 +3701,16 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_INT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && _PyLong_DigitCount((PyLongObject *)right_o) <= 1); @@ -3747,8 +3747,8 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_STR @@ -3839,7 +3839,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARA left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); + DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); @@ -3871,7 +3871,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAM left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); + DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4458,14 +4458,14 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS){ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); } // _FOR_ITER_GEN_FRAME { iter = stack_pointer[-1]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); STAT_INC(FOR_ITER, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, PyStackRef_None); @@ -4510,7 +4510,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS) // _ITER_CHECK_LIST { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER); } // _ITER_JUMP_LIST { @@ -4566,7 +4566,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS { iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); } // _ITER_JUMP_RANGE { @@ -4614,7 +4614,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS // _ITER_CHECK_TUPLE { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER); } // _ITER_JUMP_TUPLE { @@ -5071,7 +5071,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(T frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - GO_TO_INSTRUCTION(CALL_FUNCTION_EX, 1 ); + GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } } @@ -5099,7 +5099,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_ stack_pointer = _PyFrame_GetStackPointer(frame); if (err) CEVAL_GOTO(error); PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(CALL_KW, 4 ); + GO_TO_INSTRUCTION(CALL_KW); } } @@ -5323,7 +5323,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TA // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR, 2 ); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } } @@ -5917,9 +5917,9 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAM owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS @@ -5956,16 +5956,16 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); } // _GUARD_TYPE_VERSION { uint32_t type_version = read_u32(&this_instr[4].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _LOAD_ATTR_CLASS { @@ -6000,17 +6000,17 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDD PyObject *getattribute = read_obj(&this_instr[6].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR); PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( @@ -6042,14 +6042,14 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_C uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _CHECK_MANAGED_OBJECT_HAS_VALUES { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { @@ -6057,7 +6057,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_C PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = *value_ptr; - DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr_o); null = PyStackRef_NULL; @@ -6091,7 +6091,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _CHECK_ATTR_METHOD_LAZY_DICT { @@ -6099,7 +6099,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = *(PyObject **)ptr; /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(dict != NULL, LOAD_ATTR); } /* Skip 1 cache entry */ // _LOAD_ATTR_METHOD_LAZY_DICT @@ -6138,7 +6138,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_C uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_NO_DICT @@ -6178,20 +6178,20 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TA uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&this_instr[4].cache); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); } // _LOAD_ATTR_METHOD_WITH_VALUES { @@ -6230,11 +6230,11 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARA owner = stack_pointer[-1]; uint32_t dict_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR); mod_keys = keys; } // _LOAD_ATTR_MODULE_FROM_KEYS @@ -6245,11 +6245,11 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARA PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); // Clear mod_keys from stack in case we need to deopt - DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR); #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); if (!increfed) { - DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR); } #else Py_INCREF(attr_o); @@ -6285,7 +6285,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT @@ -6320,20 +6320,20 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VA uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&this_instr[4].cache); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { @@ -6362,7 +6362,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PA /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); } // _GUARD_TYPE_VERSION { @@ -6370,7 +6370,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PA uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_PROPERTY_FRAME @@ -6380,10 +6380,10 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PA assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; PyCodeObject *code = (PyCodeObject *)f->func_code; - DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - DEOPT_IF(code->co_argcount != 1, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR); + DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR); + DEOPT_IF(code->co_argcount != 1, LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); new_frame->localsplus[0] = owner; @@ -6436,7 +6436,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _LOAD_ATTR_SLOT { @@ -6444,7 +6444,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); char *addr = (char *)owner_o + index; PyObject *attr_o = *(PyObject **)addr; - DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; attr = PyStackRef_FromPyObjectNew(attr_o); @@ -6477,14 +6477,14 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_P uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _CHECK_ATTR_WITH_HINT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(dict == NULL, LOAD_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); } // _LOAD_ATTR_WITH_HINT @@ -6493,13 +6493,13 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_P PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject *attr_o; PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), LOAD_ATTR); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(ep->me_key != name, LOAD_ATTR); attr_o = ep->me_value; - DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr_o); attr = PyStackRef_FromPyObjectSteal(attr_o); @@ -6889,18 +6889,18 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_P { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(keys)); } // _GUARD_BUILTINS_VERSION_PUSH_KEYS { uint16_t version = read_u16(&this_instr[3].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); builtins_keys = keys; assert(DK_IS_UNICODE(builtins_keys)); } @@ -6909,10 +6909,10 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_P uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + DEOPT_IF(!increfed, LOAD_GLOBAL); #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -6944,9 +6944,9 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PA { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); globals_keys = keys; assert(DK_IS_UNICODE(globals_keys)); } @@ -6956,10 +6956,10 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PA uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + DEOPT_IF(!increfed, LOAD_GLOBAL); #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -7202,8 +7202,8 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -7242,8 +7242,8 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CAL PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -7847,16 +7847,16 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS){ INSTRUCTION_STATS(RESUME_CHECK); static_assert(0 == 0, "incorrect cache size"); #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME, 0); + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME, 0); + DEOPT_IF(eval_breaker != version, RESUME); #ifdef Py_GIL_DISABLED DEOPT_IF(frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME, 0); + ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME); #endif DISPATCH(); } @@ -8042,15 +8042,15 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS){ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, SEND, INLINE_CACHE_ENTRIES_SEND); + DEOPT_IF(tstate->interp->eval_frame, SEND); } // _SEND_GEN_FRAME { v = stack_pointer[-1]; receiver = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND, INLINE_CACHE_ENTRIES_SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND, INLINE_CACHE_ENTRIES_SEND); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); STAT_INC(SEND, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, v); @@ -8262,11 +8262,11 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_ uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); PyTypeObject *tp = Py_TYPE(owner_o); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(true, STORE_ATTR); } } // _GUARD_DORV_NO_DICT @@ -8277,7 +8277,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_ if (_PyObject_GetManagedDict(owner_o) || !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(true, STORE_ATTR); } } // _STORE_ATTR_INSTANCE_VALUE @@ -8322,14 +8322,14 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAM uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); } // _STORE_ATTR_SLOT { value = stack_pointer[-2]; uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; @@ -8361,7 +8361,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); } // _STORE_ATTR_WITH_HINT { @@ -8370,12 +8370,12 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); - DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(dict == NULL, STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR); #ifdef Py_GIL_DISABLED if (dict != _PyObject_GetManagedDict(owner_o)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(true, STORE_ATTR); } #endif assert(PyDict_CheckExact((PyObject *)dict)); @@ -8383,17 +8383,17 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_ if (hint >= (size_t)dict->ma_keys->dk_nentries || !DK_IS_UNICODE(dict->ma_keys)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(true, STORE_ATTR); } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(true, STORE_ATTR); } PyObject *old_value = ep->me_value; if (old_value == NULL) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(true, STORE_ATTR); } _PyFrame_SetStackPointer(frame, stack_pointer); _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); @@ -8663,7 +8663,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PAR dict_st = stack_pointer[-2]; value = stack_pointer[-3]; PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, @@ -8695,16 +8695,16 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL value = stack_pointer[-3]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR); // Ensure index < len(list) if (index >= PyList_GET_SIZE(list)) { UNLOCK_OBJECT(list); - DEOPT_IF(true, STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + DEOPT_IF(true, STORE_SUBSCR); } STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); @@ -8804,7 +8804,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_P uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL); } // _REPLACE_WITH_TRUE { @@ -8829,7 +8829,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS){ /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; - DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL); STAT_INC(TO_BOOL, hit); DISPATCH(); } @@ -8849,7 +8849,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS){ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value_o)) { assert(_Py_IsImmortal(value_o)); @@ -8878,7 +8878,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS){ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL); STAT_INC(TO_BOOL, hit); res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; PyStackRef_CLOSE(value); @@ -8901,7 +8901,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS){ /* Skip 2 cache entries */ value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL); STAT_INC(TO_BOOL, hit); res = PyStackRef_False; stack_pointer[-1] = res; @@ -8923,7 +8923,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS){ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL); STAT_INC(TO_BOOL, hit); if (value_o == &_Py_STR(empty)) { assert(_Py_IsImmortal(value_o)); @@ -9083,11 +9083,11 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); - DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE); if (PyList_GET_SIZE(seq_o) != oparg) { UNLOCK_OBJECT(seq_o); - DEOPT_IF(true, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(true, UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq_o); @@ -9116,8 +9116,8 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { @@ -9145,8 +9145,8 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_ seq = stack_pointer[-1]; assert(oparg == 2); PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 64098cf5d9332a..30af5beb51e010 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -154,7 +154,6 @@ def deopt_if( assert inst is not None assert inst.family is not None self.out.emit(inst.family.name) - self.out.emit(f", {inst.family.size}") self.out.emit(");\n") return not always_true(first_tkn) From 5615d6f39e7b4ab0b4df9a00ea83ccf089250652 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 10 Jan 2025 10:13:21 +0800 Subject: [PATCH 020/110] Add unknown opcode handlers --- Python/generated_tail_call_handlers.c.h | 49 +++++++++++++++++++ .../tier1_tail_call_generator.py | 20 +++++++- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 588fc5ee439bd9..1f326199735c47 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -9265,6 +9265,15 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS){ } } +Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS){ + +int opcode = next_instr->op.code; +_PyErr_Format(tstate, PyExc_SystemError, + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); + CEVAL_GOTO(error);} static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [BINARY_OP] = _TAIL_CALL_BINARY_OP, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, @@ -9482,5 +9491,45 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, + [117] = _TAIL_CALL_UNKNOWN_OPCODE, + [118] = _TAIL_CALL_UNKNOWN_OPCODE, + [119] = _TAIL_CALL_UNKNOWN_OPCODE, + [120] = _TAIL_CALL_UNKNOWN_OPCODE, + [121] = _TAIL_CALL_UNKNOWN_OPCODE, + [122] = _TAIL_CALL_UNKNOWN_OPCODE, + [123] = _TAIL_CALL_UNKNOWN_OPCODE, + [124] = _TAIL_CALL_UNKNOWN_OPCODE, + [125] = _TAIL_CALL_UNKNOWN_OPCODE, + [126] = _TAIL_CALL_UNKNOWN_OPCODE, + [127] = _TAIL_CALL_UNKNOWN_OPCODE, + [128] = _TAIL_CALL_UNKNOWN_OPCODE, + [129] = _TAIL_CALL_UNKNOWN_OPCODE, + [130] = _TAIL_CALL_UNKNOWN_OPCODE, + [131] = _TAIL_CALL_UNKNOWN_OPCODE, + [132] = _TAIL_CALL_UNKNOWN_OPCODE, + [133] = _TAIL_CALL_UNKNOWN_OPCODE, + [134] = _TAIL_CALL_UNKNOWN_OPCODE, + [135] = _TAIL_CALL_UNKNOWN_OPCODE, + [136] = _TAIL_CALL_UNKNOWN_OPCODE, + [137] = _TAIL_CALL_UNKNOWN_OPCODE, + [138] = _TAIL_CALL_UNKNOWN_OPCODE, + [139] = _TAIL_CALL_UNKNOWN_OPCODE, + [140] = _TAIL_CALL_UNKNOWN_OPCODE, + [141] = _TAIL_CALL_UNKNOWN_OPCODE, + [142] = _TAIL_CALL_UNKNOWN_OPCODE, + [143] = _TAIL_CALL_UNKNOWN_OPCODE, + [144] = _TAIL_CALL_UNKNOWN_OPCODE, + [145] = _TAIL_CALL_UNKNOWN_OPCODE, + [146] = _TAIL_CALL_UNKNOWN_OPCODE, + [147] = _TAIL_CALL_UNKNOWN_OPCODE, + [148] = _TAIL_CALL_UNKNOWN_OPCODE, + [228] = _TAIL_CALL_UNKNOWN_OPCODE, + [229] = _TAIL_CALL_UNKNOWN_OPCODE, + [230] = _TAIL_CALL_UNKNOWN_OPCODE, + [231] = _TAIL_CALL_UNKNOWN_OPCODE, + [232] = _TAIL_CALL_UNKNOWN_OPCODE, + [233] = _TAIL_CALL_UNKNOWN_OPCODE, + [234] = _TAIL_CALL_UNKNOWN_OPCODE, + [235] = _TAIL_CALL_UNKNOWN_OPCODE, }; #undef TIER_ONE diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index 3b5761a3238314..9416f99d3642c8 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -82,8 +82,6 @@ def generate_tier1( out.emit("static inline PyObject *_TAIL_CALL_shim(TAIL_CALL_PARAMS);\n") out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256];\n"); - # Emit error handlers - generate_label_handlers(outfile) emitter = Emitter(out) @@ -105,9 +103,27 @@ def generate_tier1( out.emit("\n") + # Emit unknown opcode handler. + out.emit(function_proto("UNKNOWN_OPCODE")) + out.emit("{\n") + out.emit(""" +int opcode = next_instr->op.code; +_PyErr_Format(tstate, PyExc_SystemError, + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); +""") + out.emit("CEVAL_GOTO(error);") + out.emit("}\n") + out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256] = {\n") for name in sorted(analysis.instructions.keys()): out.emit(f"[{name}] = _TAIL_CALL_{name},\n") + named_values = analysis.opmap.values() + for rest in range(256): + if rest not in named_values: + out.emit(f"[{rest}] = _TAIL_CALL_UNKNOWN_OPCODE,\n") out.emit("};\n") outfile.write(FOOTER) From 71eba582ef5af7ff8b0723e3a8595bceea14f255 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 10 Jan 2025 10:37:38 +0800 Subject: [PATCH 021/110] cleanup --- Include/Python.h | 1 + Python/bytecodes.c | 4 +++- Python/ceval.c | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Include/Python.h b/Include/Python.h index 88248cb4e481e6..64be80145890a3 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -8,6 +8,7 @@ // Since this is a "meta-include" file, "#ifdef __cplusplus / extern "C" {" // is not needed. + // Include Python header files #include "patchlevel.h" #include "pyconfig.h" diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 97d9430a96749b..c17ca8099118b0 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1304,7 +1304,9 @@ dummy_func( tier1 inst(CLEANUP_THROW, (sub_iter_st, last_sent_val_st, exc_value_st -- none, value)) { PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - // assert(throwflag); + #ifndef Py_TAIL_CALL_INTERP + assert(throwflag); + #endif assert(exc_value && PyExceptionInstance_Check(exc_value)); int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); diff --git a/Python/ceval.c b/Python/ceval.c index 3cdea3f3512abf..2c182989f42b24 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1195,7 +1195,7 @@ TAIL_CALL_TARGET(resume_with_error): #endif // _Py_TIER2 -} /* _PyEval_EvalFrameDefault */ +} #ifdef DO_NOT_OPTIMIZE_INTERP_LOOP # pragma optimize("", on) From e1d5f41c5f8fba38e61b48835726195cf37642bc Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 10 Jan 2025 11:04:18 +0800 Subject: [PATCH 022/110] Add to makefile --- Makefile.pre.in | 1 + Python/ceval.c | 2 ++ 2 files changed, 3 insertions(+) diff --git a/Makefile.pre.in b/Makefile.pre.in index 0f2f144372d6b1..9aba7881f5973f 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2061,6 +2061,7 @@ Python/ceval.o: \ $(srcdir)/Python/ceval_macros.h \ $(srcdir)/Python/condvar.h \ $(srcdir)/Python/generated_cases.c.h \ + $(srcdir)/Python/generated_tail_call_handlers.c.h \ $(srcdir)/Python/executor_cases.c.h \ $(srcdir)/Python/opcode_targets.h diff --git a/Python/ceval.c b/Python/ceval.c index 2c182989f42b24..71027ead4cc053 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -803,8 +803,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #ifdef Py_STATS int lastopcode = 0; #endif +#ifndef Py_TAIL_CALL_INTERP uint8_t opcode; /* Current opcode */ int oparg; /* Current opcode argument, if any */ +#endif #ifdef LLTRACE int lltrace = 0; #endif From b57788857576536b8ede64db13ba92980ad90480 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 10 Jan 2025 11:14:57 +0800 Subject: [PATCH 023/110] fix debug --- Python/ceval.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 71027ead4cc053..70a9809edd9eaf 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -785,7 +785,11 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) #include "generated_tail_call_handlers.c.h" static inline PyObject *_TAIL_CALL_shim(TAIL_CALL_PARAMS) { +#ifdef LLTRACE + return (INSTRUCTION_TABLE[next_instr->op.code])(frame, stack_pointer, tstate, next_instr, next_instr->op.arg, entry_frame, lltrace); +#else return (INSTRUCTION_TABLE[next_instr->op.code])(frame, stack_pointer, tstate, next_instr, next_instr->op.arg, entry_frame); +#endif } #endif @@ -818,11 +822,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #if defined(Py_DEBUG) && !defined(Py_STACKREF_DEBUG) /* Set these to invalid but identifiable values for debugging. */ - e.f_funcobj = (_PyStackRef){.bits = 0xaaa0}; - e.f_locals = (PyObject*)0xaaa1; - e.frame_obj = (PyFrameObject*)0xaaa2; - e.f_globals = (PyObject*)0xaaa3; - e.f_builtins = (PyObject*)0xaaa4; + entry_f.f_funcobj = (_PyStackRef){.bits = 0xaaa0}; + entry_f.f_locals = (PyObject*)0xaaa1; + entry_f.frame_obj = (PyFrameObject*)0xaaa2; + entry_f.f_globals = (PyObject*)0xaaa3; + entry_f.f_builtins = (PyObject*)0xaaa4; #endif entry_f.f_executable = PyStackRef_None; entry_f.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; From ac80cdd1cc7415d14ebcea749e7b205cd4f12a7d Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 10 Jan 2025 22:47:55 +0800 Subject: [PATCH 024/110] Fix up generation --- Python/ceval.c | 10 +- Python/generated_cases.c.h | 4 +- Python/generated_tail_call_handlers.c.h | 498 +++++++++--------- .../tier1_tail_call_generator.py | 7 +- 4 files changed, 276 insertions(+), 243 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 70a9809edd9eaf..40d1031cdd19b7 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1028,11 +1028,15 @@ TAIL_CALL_TARGET(exception_unwind): #endif #ifdef Py_TAIL_CALL_INTERP -#ifdef LLTRACE +# ifdef IN_TAIL_CALL_INTERP + DISPATCH(); +# else +# ifdef LLTRACE return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, entry_frame, lltrace); -#else +# else return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, entry_frame); -#endif +# endif +# endif #else DISPATCH(); #endif diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 439f1424f3465e..2a3657df54a304 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3187,7 +3187,9 @@ last_sent_val_st = stack_pointer[-2]; sub_iter_st = stack_pointer[-3]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - // assert(throwflag); + #ifndef Py_TAIL_CALL_INTERP + assert(throwflag); + #endif assert(exc_value && PyExceptionInstance_Check(exc_value)); _PyFrame_SetStackPointer(frame, stack_pointer); int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 1f326199735c47..c2243513a1c5f1 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -7,38 +7,49 @@ #error "This file is for tail-calling interpreter only." #endif #define TIER_ONE 1 +#define IN_TAIL_CALL_INTERP 1 static inline PyObject *_TAIL_CALL_shim(TAIL_CALL_PARAMS); static py_tail_call_funcptr INSTRUCTION_TABLE[256]; -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_pop_4_error(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_pop_3_error(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_error(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_resume_with_error(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_pop_4_error(TAIL_CALL_PARAMS) +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_4_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_3_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_resume_with_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_4_error(TAIL_CALL_PARAMS) { + int opcode = next_instr->op.code; + (void)opcode; STACK_SHRINK(1); CEVAL_GOTO(pop_3_error); } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_pop_3_error(TAIL_CALL_PARAMS) +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_3_error(TAIL_CALL_PARAMS) { + int opcode = next_instr->op.code; + (void)opcode; STACK_SHRINK(1); CEVAL_GOTO(pop_2_error); } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS) +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS) { + int opcode = next_instr->op.code; + (void)opcode; STACK_SHRINK(1); CEVAL_GOTO(pop_1_error); } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS) +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS) { + int opcode = next_instr->op.code; + (void)opcode; STACK_SHRINK(1); CEVAL_GOTO(error); } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_error(TAIL_CALL_PARAMS) +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS) { + int opcode = next_instr->op.code; + (void)opcode; /* Double-check exception status. */ #ifdef NDEBUG if (!_PyErr_Occurred(tstate)) { @@ -60,8 +71,10 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_error(TAIL_CALL_PARAMS) _PyEval_MonitorRaise(tstate, frame, next_instr-1); CEVAL_GOTO(exception_unwind); } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS) +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS) { + int opcode = next_instr->op.code; + (void)opcode; { /* We can't use frame->instr_ptr here, as RERAISE may have set it */ int offset = INSTR_OFFSET()-1; @@ -114,11 +127,15 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_exception_unwind(TAIL_CALL_PARA #endif #ifdef Py_TAIL_CALL_INTERP - #ifdef LLTRACE + # ifdef IN_TAIL_CALL_INTERP + DISPATCH(); + # else + # ifdef LLTRACE return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, entry_frame, lltrace); - #else + # else return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, entry_frame); - #endif + # endif + # endif #else DISPATCH(); #endif @@ -126,8 +143,10 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_exception_unwind(TAIL_CALL_PARA CEVAL_GOTO(exit_unwind); } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS) +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS) { + int opcode = next_instr->op.code; + (void)opcode; assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); assert(frame != entry_frame); @@ -145,8 +164,10 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS) CEVAL_GOTO(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_resume_with_error(TAIL_CALL_PARAMS) +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_resume_with_error(TAIL_CALL_PARAMS) { + int opcode = next_instr->op.code; + (void)opcode; next_instr = frame->instr_ptr; stack_pointer = _PyFrame_GetStackPointer(frame); CEVAL_GOTO(error); @@ -155,7 +176,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_resume_with_error(TAIL_CALL_PAR } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -208,7 +229,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -248,7 +269,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_P } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -287,7 +308,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PAR } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -326,7 +347,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -391,7 +412,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(T } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -431,7 +452,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_C } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -470,7 +491,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CAL } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -510,7 +531,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_C } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -549,7 +570,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CAL } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -603,7 +624,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -654,7 +675,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS) } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -693,7 +714,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -756,7 +777,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -800,7 +821,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CAL } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -836,7 +857,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -872,7 +893,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -896,7 +917,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -940,7 +961,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -988,7 +1009,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -1022,7 +1043,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -1061,7 +1082,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -1085,7 +1106,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CACHE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -1098,7 +1119,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CACHE(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -1260,7 +1281,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -1362,7 +1383,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -1469,7 +1490,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -1572,7 +1593,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -1653,7 +1674,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -1740,7 +1761,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PAR } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -1828,7 +1849,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -1900,7 +1921,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -2070,7 +2091,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -2092,7 +2113,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -2121,7 +2142,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -2169,7 +2190,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAM } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -2330,7 +2351,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -2439,7 +2460,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -2535,7 +2556,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -2623,7 +2644,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -2674,7 +2695,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -2716,7 +2737,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -2805,7 +2826,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAI } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -2894,7 +2915,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WIT } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -2970,7 +2991,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(T } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -3049,7 +3070,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_C } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -3139,7 +3160,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_P } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -3224,7 +3245,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -3306,7 +3327,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAM } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -3361,7 +3382,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -3416,7 +3437,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -3448,7 +3469,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -3499,7 +3520,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -3531,7 +3552,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAM } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -3548,7 +3569,9 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS) last_sent_val_st = stack_pointer[-2]; sub_iter_st = stack_pointer[-3]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - // assert(throwflag); + #ifndef Py_TAIL_CALL_INTERP + assert(throwflag); + #endif assert(exc_value && PyExceptionInstance_Check(exc_value)); _PyFrame_SetStackPointer(frame, stack_pointer); int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); @@ -3577,7 +3600,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS) } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -3642,7 +3665,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -3684,7 +3707,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -3730,7 +3753,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -3773,7 +3796,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -3823,7 +3846,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -3855,7 +3878,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -3888,7 +3911,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAM } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -3912,7 +3935,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS) } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_COPY(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -3931,7 +3954,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_COPY(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -3953,7 +3976,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -3974,7 +3997,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -3996,7 +4019,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4018,7 +4041,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4044,7 +4067,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS) } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4077,7 +4100,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4102,7 +4125,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS) } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4135,7 +4158,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4172,7 +4195,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4207,7 +4230,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS) } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_END_FOR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4223,7 +4246,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_END_FOR(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_END_SEND(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4245,7 +4268,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_END_SEND(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4282,7 +4305,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4306,7 +4329,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAM } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4321,7 +4344,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4350,7 +4373,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS) } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4376,7 +4399,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4444,7 +4467,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4496,7 +4519,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4551,7 +4574,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS) } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4600,7 +4623,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4652,7 +4675,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4703,7 +4726,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4727,7 +4750,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4748,7 +4771,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS) } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4770,7 +4793,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4795,7 +4818,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4842,7 +4865,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_P } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4865,7 +4888,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -4894,7 +4917,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5064,7 +5087,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PAR } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5075,7 +5098,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(T } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5103,7 +5126,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5132,7 +5155,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5163,7 +5186,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5206,7 +5229,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5229,7 +5252,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_C } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5257,7 +5280,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5270,7 +5293,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5311,7 +5334,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PAR } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5327,7 +5350,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5340,7 +5363,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CAL } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5360,7 +5383,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE( } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5382,7 +5405,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(T } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5402,7 +5425,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NO } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5422,7 +5445,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(T } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5498,7 +5521,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_P } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5547,7 +5570,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5620,7 +5643,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_C } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5643,7 +5666,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_IS_OP(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5666,7 +5689,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_IS_OP(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5728,7 +5751,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS) } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5745,7 +5768,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5757,7 +5780,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5777,7 +5800,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5818,7 +5841,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5900,7 +5923,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5939,7 +5962,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAM } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -5984,7 +6007,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6024,7 +6047,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDD } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6073,7 +6096,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_C } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6120,7 +6143,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6160,7 +6183,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_C } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6212,7 +6235,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6268,7 +6291,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6303,7 +6326,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6349,7 +6372,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6418,7 +6441,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6459,7 +6482,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6515,7 +6538,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_P } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6543,7 +6566,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6569,7 +6592,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6586,7 +6609,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6605,7 +6628,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_P } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6629,7 +6652,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6646,7 +6669,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6664,7 +6687,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_P } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6690,7 +6713,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAM } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6711,7 +6734,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_P } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6750,7 +6773,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6824,7 +6847,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6873,7 +6896,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6928,7 +6951,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_P } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6975,7 +6998,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -6999,7 +7022,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7020,7 +7043,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7038,7 +7061,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7079,7 +7102,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7182,7 +7205,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAM } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7221,7 +7244,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7273,7 +7296,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CAL } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7292,7 +7315,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7317,7 +7340,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS) } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7348,7 +7371,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7390,7 +7413,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7416,7 +7439,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7435,7 +7458,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS) } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7454,7 +7477,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_NOP(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7465,7 +7488,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_NOP(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7476,7 +7499,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7497,7 +7520,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7518,7 +7541,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PAR } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7555,7 +7578,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7592,7 +7615,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7613,7 +7636,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7629,7 +7652,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7658,7 +7681,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS) } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7674,7 +7697,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7703,7 +7726,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS) } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_RERAISE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7751,7 +7774,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_RERAISE(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_RESERVED(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7764,7 +7787,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_RESERVED(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_RESUME(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7838,7 +7861,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_RESUME(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7862,7 +7885,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7899,7 +7922,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARA } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -7933,7 +7956,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_SEND(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8027,7 +8050,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_SEND(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8083,7 +8106,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8122,7 +8145,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PAR } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8145,7 +8168,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8173,7 +8196,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CAL } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8196,7 +8219,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8245,7 +8268,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8305,7 +8328,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8344,7 +8367,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAM } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8412,7 +8435,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8431,7 +8454,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8447,7 +8470,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8466,7 +8489,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8487,7 +8510,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8508,7 +8531,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8546,7 +8569,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8598,7 +8621,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8647,7 +8670,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8678,7 +8701,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PAR } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8720,7 +8743,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_SWAP(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8742,7 +8765,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_SWAP(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8786,7 +8809,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8817,7 +8840,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_P } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8835,7 +8858,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8864,7 +8887,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8887,7 +8910,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8909,7 +8932,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8939,7 +8962,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8960,7 +8983,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8981,7 +9004,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -8999,7 +9022,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -9022,7 +9045,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -9069,7 +9092,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAM } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -9102,7 +9125,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -9130,7 +9153,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -9159,7 +9182,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -9213,7 +9236,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PAR } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; (void)(opcode); { @@ -9265,7 +9288,7 @@ Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS){ } } -Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS){ int opcode = next_instr->op.code; _PyErr_Format(tstate, PyExc_SystemError, @@ -9533,3 +9556,4 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [235] = _TAIL_CALL_UNKNOWN_OPCODE, }; #undef TIER_ONE +#undef IN_TAIL_CALL_INTERP diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index 9416f99d3642c8..9cd9c13862b8d5 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -29,7 +29,7 @@ DEFAULT_CEVAL_INPUT = ROOT / "Python/ceval.c" -FOOTER = "#undef TIER_ONE\n" +FOOTER = "#undef TIER_ONE\n#undef IN_TAIL_CALL_INTERP\n" TARGET_LABEL = "TAIL_CALL_TARGET" @@ -50,6 +50,8 @@ def generate_label_handlers(outfile: TextIO): out.emit(function_proto(curr_proto)) out.emit("\n") out.emit("{\n") + out.emit("int opcode = next_instr->op.code;\n") + out.emit("(void)opcode;\n") for line in lines: if TARGET_LABEL in line: break @@ -64,7 +66,7 @@ def generate_label_handlers(outfile: TextIO): def function_proto(name: str) -> str: - return f"Py_PRESERVE_NONE_CC static PyObject * _TAIL_CALL_{name}(TAIL_CALL_PARAMS)" + return f"Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_{name}(TAIL_CALL_PARAMS)" def generate_tier1( filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool @@ -76,6 +78,7 @@ def generate_tier1( #error "This file is for tail-calling interpreter only." #endif #define TIER_ONE 1 +#define IN_TAIL_CALL_INTERP 1 """ ) out = CWriter(outfile, 0, lines) From 6ae3e03c0a0d7ce2cd821e7fc62a6e293a042282 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 11 Jan 2025 00:15:02 +0800 Subject: [PATCH 025/110] fix instrumented instructions partially --- Python/generated_tail_call_handlers.c.h | 43 ++++++++++--------- .../tier1_tail_call_generator.py | 5 ++- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index c2243513a1c5f1..001189db22cc3d 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -4269,7 +4269,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS){ } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; + int opcode = ENTER_EXECUTOR; (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -4918,7 +4918,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS){ } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_CALL; (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -5088,7 +5088,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARA } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_CALL_FUNCTION_EX; (void)(opcode); { frame->instr_ptr = next_instr; @@ -5099,7 +5099,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TA } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_CALL_KW; (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -5127,7 +5127,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_P } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_END_FOR; (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -5156,7 +5156,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_P } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_END_SEND; (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -5187,7 +5187,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_ } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_FOR_ITER; (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -5230,7 +5230,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_ } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_INSTRUCTION; (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -5253,7 +5253,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CA } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_JUMP_BACKWARD; (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -5281,7 +5281,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_ } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_JUMP_FORWARD; (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -5294,7 +5294,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_C } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_LINE; (void)(opcode); { _Py_CODEUNIT* const prev_instr = frame->instr_ptr; @@ -5335,7 +5335,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARA } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_LOAD_SUPER_ATTR; (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -5351,7 +5351,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAI } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_NOT_TAKEN; (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -5364,7 +5364,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -5384,7 +5384,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(T } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_POP_JUMP_IF_NONE; (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -5406,7 +5406,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TA } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -5426,7 +5426,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NON } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -5446,7 +5446,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TA } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_RESUME; (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -5522,7 +5522,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PA } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_RETURN_VALUE; (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -5571,7 +5571,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_C } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_YIELD_VALUE; (void)(opcode); { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -7795,7 +7795,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS){ next_instr += 1; INSTRUCTION_STATS(RESUME); PREDICTED(RESUME); - _Py_CODEUNIT* const this_instr = next_instr - 1; + _Py_CODEUNIT* this_instr = next_instr - 1; (void)this_instr; // _LOAD_BYTECODE { @@ -7832,6 +7832,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS){ CEVAL_GOTO(error); } next_instr = this_instr; + DISPATCH(); } } diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index 9cd9c13862b8d5..def253a18a9007 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -93,7 +93,10 @@ def generate_tier1( out.emit("\n") out.emit(function_proto(name)) out.emit("{\n") - out.emit("int opcode = next_instr->op.code;\n") + if analysis.opmap[name] >= analysis.min_instrumented: + out.emit(f"int opcode = {name};\n") + else: + out.emit("int opcode = next_instr->op.code;\n") # Some instructions don't use opcode. out.emit(f"(void)(opcode);\n") out.emit("{\n") From a27ee2d191be8a728f2d80b09c7aa1214f0a504a Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 11 Jan 2025 00:41:36 +0800 Subject: [PATCH 026/110] Fix instrumentation completely --- Python/ceval.c | 13 +- Python/ceval_macros.h | 14 +- Python/generated_tail_call_handlers.c.h | 15002 ++++++++-------- .../tier1_tail_call_generator.py | 11 - 4 files changed, 7074 insertions(+), 7966 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 40d1031cdd19b7..4d3414e3d99018 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -785,10 +785,11 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) #include "generated_tail_call_handlers.c.h" static inline PyObject *_TAIL_CALL_shim(TAIL_CALL_PARAMS) { + opcode = next_instr->op.code; #ifdef LLTRACE - return (INSTRUCTION_TABLE[next_instr->op.code])(frame, stack_pointer, tstate, next_instr, next_instr->op.arg, entry_frame, lltrace); + return (INSTRUCTION_TABLE[opcode])(frame, stack_pointer, tstate, next_instr, opcode, next_instr->op.arg, entry_frame, lltrace); #else - return (INSTRUCTION_TABLE[next_instr->op.code])(frame, stack_pointer, tstate, next_instr, next_instr->op.arg, entry_frame); + return (INSTRUCTION_TABLE[opcode])(frame, stack_pointer, tstate, next_instr, opcode, next_instr->op.arg, entry_frame); #endif } #endif @@ -908,9 +909,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #ifdef Py_TAIL_CALL_INTERP #ifdef LLTRACE - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, entry_frame, lltrace); + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0, entry_frame, lltrace); #else - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, entry_frame); + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0, entry_frame); #endif #else DISPATCH(); @@ -1032,9 +1033,9 @@ TAIL_CALL_TARGET(exception_unwind): DISPATCH(); # else # ifdef LLTRACE - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, entry_frame, lltrace); + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0, entry_frame, lltrace); # else - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, entry_frame); + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0, entry_frame); # endif # endif #else diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 27aeb98fb2bf00..39e22768c2d720 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -71,11 +71,11 @@ #endif #ifdef LLTRACE -# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame *entry_frame, int lltrace -# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, oparg, entry_frame, lltrace +# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int opcode, int oparg, _PyInterpreterFrame *entry_frame, int lltrace +# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, opcode, oparg, entry_frame, lltrace #else -# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, _PyInterpreterFrame *entry_frame -# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, oparg, entry_frame +# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int opcode, int oparg, _PyInterpreterFrame *entry_frame +# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, opcode, oparg, entry_frame #endif #ifdef Py_TAIL_CALL_INTERP @@ -331,12 +331,12 @@ GETITEM(PyObject *v, Py_ssize_t i) { #ifdef LLTRACE #define GO_TO_INSTRUCTION(op) do { \ Py_MUSTTAIL \ - return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - _PyOpcode_Caches[_PyOpcode_Deopt[op]], oparg, entry_frame, lltrace); \ + return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - _PyOpcode_Caches[_PyOpcode_Deopt[op]], opcode, oparg, entry_frame, lltrace); \ } while (0) #else -#define GO_TO_INSTRUCTION(op) do { \ +#define GO_TO_INSTRUCTION(op) do { \ Py_MUSTTAIL \ - return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - _PyOpcode_Caches[_PyOpcode_Deopt[op]], oparg, entry_frame); \ + return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - _PyOpcode_Caches[_PyOpcode_Deopt[op]], opcode, oparg, entry_frame); \ } while (0) #endif #else diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 001189db22cc3d..26a6a481464190 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -20,36 +20,26 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_resume_with_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_4_error(TAIL_CALL_PARAMS) { - int opcode = next_instr->op.code; - (void)opcode; STACK_SHRINK(1); CEVAL_GOTO(pop_3_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_3_error(TAIL_CALL_PARAMS) { - int opcode = next_instr->op.code; - (void)opcode; STACK_SHRINK(1); CEVAL_GOTO(pop_2_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS) { - int opcode = next_instr->op.code; - (void)opcode; STACK_SHRINK(1); CEVAL_GOTO(pop_1_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS) { - int opcode = next_instr->op.code; - (void)opcode; STACK_SHRINK(1); CEVAL_GOTO(error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS) { - int opcode = next_instr->op.code; - (void)opcode; /* Double-check exception status. */ #ifdef NDEBUG if (!_PyErr_Occurred(tstate)) { @@ -73,8 +63,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS) } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS) { - int opcode = next_instr->op.code; - (void)opcode; { /* We can't use frame->instr_ptr here, as RERAISE may have set it */ int offset = INSTR_OFFSET()-1; @@ -131,9 +119,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAM DISPATCH(); # else # ifdef LLTRACE - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, entry_frame, lltrace); + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0, entry_frame, lltrace); # else - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, entry_frame); + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0, entry_frame); # endif # endif #else @@ -145,8 +133,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAM } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS) { - int opcode = next_instr->op.code; - (void)opcode; assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); assert(frame != entry_frame); @@ -166,8 +152,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS) } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_resume_with_error(TAIL_CALL_PARAMS) { - int opcode = next_instr->op.code; - (void)opcode; next_instr = frame->instr_ptr; stack_pointer = _PyFrame_GetStackPointer(frame); CEVAL_GOTO(error); @@ -177,3280 +161,2977 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_resume_with_error(TAIL_CALL_PARA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP); - PREDICTED(BINARY_OP); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef lhs; - _PyStackRef rhs; - _PyStackRef res; - // _SPECIALIZE_BINARY_OP - { - rhs = stack_pointer[-1]; - lhs = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(BINARY_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - assert(NB_ADD <= oparg); - assert(oparg <= NB_INPLACE_XOR); - } - // _BINARY_OP - { - PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); - PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); - assert(_PyEval_BinaryOps[oparg]); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP); + PREDICTED(BINARY_OP); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef lhs; + _PyStackRef rhs; + _PyStackRef res; + // _SPECIALIZE_BINARY_OP + { + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(lhs); - PyStackRef_CLOSE(rhs); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - res = PyStackRef_FromPyObjectSteal(res_o); + DISPATCH_SAME_OPARG(); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + OPCODE_DEFERRED_INC(BINARY_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + assert(NB_ADD <= oparg); + assert(oparg <= NB_INPLACE_XOR); + } + // _BINARY_OP + { + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); + assert(_PyEval_BinaryOps[oparg]); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(lhs); + PyStackRef_CLOSE(rhs); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); + res = PyStackRef_FromPyObjectSteal(res_o); } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_FLOAT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); - } - /* Skip 1 cache entry */ - // _BINARY_OP_ADD_FLOAT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval + - ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_FLOAT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_ADD_FLOAT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); + res = PyStackRef_FromPyObjectSteal(res_o); } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_ADD_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_INT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); - } - /* Skip 1 cache entry */ - // _BINARY_OP_ADD_INT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_INT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); } + /* Skip 1 cache entry */ + // _BINARY_OP_ADD_INT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_UNICODE - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); - } - /* Skip 1 cache entry */ - // _BINARY_OP_ADD_UNICODE - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyUnicode_Concat(left_o, right_o); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_UNICODE + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_ADD_UNICODE + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); + res = PyStackRef_FromPyObjectSteal(res_o); } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - // _GUARD_BOTH_UNICODE - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); - } - /* Skip 1 cache entry */ - // _BINARY_OP_INPLACE_ADD_UNICODE - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - int next_oparg; - #if TIER_ONE - assert(next_instr->op.code == STORE_FAST); - next_oparg = next_instr->op.arg; - #else - next_oparg = CURRENT_OPERAND0(); - #endif - _PyStackRef *target_local = &GETLOCAL(next_oparg); - DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP); - STAT_INC(BINARY_OP, hit); - /* Handle `left = left + right` or `left += right` for str. - * - * When possible, extend `left` in place rather than - * allocating a new PyUnicodeObject. This attempts to avoid - * quadratic behavior when one neglects to use str.join(). - * - * If `left` has only two references remaining (one from - * the stack, one in the locals), DECREFing `left` leaves - * only the locals reference, so PyUnicode_Append knows - * that the string is safe to mutate. - */ - assert(Py_REFCNT(left_o) >= 2); - PyStackRef_CLOSE(left); - PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); - PyUnicode_Append(&temp, right_o); - *target_local = PyStackRef_FromPyObjectSteal(temp); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (PyStackRef_IsNull(*target_local)) CEVAL_GOTO(pop_2_error); - #if TIER_ONE - // The STORE_FAST is already done. This is done here in tier one, - // and during trace projection in tier two: - assert(next_instr->op.code == STORE_FAST); - SKIP_OVER(1); - #endif - } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + // _GUARD_BOTH_UNICODE + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_INPLACE_ADD_UNICODE + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + int next_oparg; + #if TIER_ONE + assert(next_instr->op.code == STORE_FAST); + next_oparg = next_instr->op.arg; + #else + next_oparg = CURRENT_OPERAND0(); + #endif + _PyStackRef *target_local = &GETLOCAL(next_oparg); + DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP); + STAT_INC(BINARY_OP, hit); + /* Handle `left = left + right` or `left += right` for str. + * + * When possible, extend `left` in place rather than + * allocating a new PyUnicodeObject. This attempts to avoid + * quadratic behavior when one neglects to use str.join(). + * + * If `left` has only two references remaining (one from + * the stack, one in the locals), DECREFing `left` leaves + * only the locals reference, so PyUnicode_Append knows + * that the string is safe to mutate. + */ + assert(Py_REFCNT(left_o) >= 2); + PyStackRef_CLOSE(left); + PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); + PyUnicode_Append(&temp, right_o); + *target_local = PyStackRef_FromPyObjectSteal(temp); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (PyStackRef_IsNull(*target_local)) CEVAL_GOTO(pop_2_error); + #if TIER_ONE + // The STORE_FAST is already done. This is done here in tier one, + // and during trace projection in tier two: + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); + #endif } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_FLOAT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); - } - /* Skip 1 cache entry */ - // _BINARY_OP_MULTIPLY_FLOAT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval * - ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_FLOAT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_MULTIPLY_FLOAT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); + res = PyStackRef_FromPyObjectSteal(res_o); } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_INT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); - } - /* Skip 1 cache entry */ - // _BINARY_OP_MULTIPLY_INT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_INT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_MULTIPLY_INT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); + res = PyStackRef_FromPyObjectSteal(res_o); } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_FLOAT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); - } - /* Skip 1 cache entry */ - // _BINARY_OP_SUBTRACT_FLOAT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval - - ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_FLOAT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_SUBTRACT_FLOAT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); + res = PyStackRef_FromPyObjectSteal(res_o); } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_INT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); - } - /* Skip 1 cache entry */ - // _BINARY_OP_SUBTRACT_INT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_INT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_SUBTRACT_INT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); + res = PyStackRef_FromPyObjectSteal(res_o); } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BINARY_SLICE); - _PyStackRef container; - _PyStackRef start; - _PyStackRef stop; - _PyStackRef res; - // _SPECIALIZE_BINARY_SLICE - { - // Placeholder until we implement BINARY_SLICE specialization - #if ENABLE_SPECIALIZATION - OPCODE_DEFERRED_INC(BINARY_SLICE); - #endif /* ENABLE_SPECIALIZATION */ + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BINARY_SLICE); + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + _PyStackRef res; + // _SPECIALIZE_BINARY_SLICE + { + // Placeholder until we implement BINARY_SLICE specialization + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(BINARY_SLICE); + #endif /* ENABLE_SPECIALIZATION */ + } + // _BINARY_SLICE + { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyObject *res_o; + // Can't use ERROR_IF() here, because we haven't + // DECREF'ed container yet, and we still own slice. + if (slice == NULL) { + res_o = NULL; } - // _BINARY_SLICE - { - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; + else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); + res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); stack_pointer = _PyFrame_GetStackPointer(frame); - PyObject *res_o; - // Can't use ERROR_IF() here, because we haven't - // DECREF'ed container yet, and we still own slice. - if (slice == NULL) { - res_o = NULL; - } - else { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); - stack_pointer = _PyFrame_GetStackPointer(frame); - Py_DECREF(slice); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - } - PyStackRef_CLOSE(container); - if (res_o == NULL) CEVAL_GOTO(pop_3_error); - res = PyStackRef_FromPyObjectSteal(res_o); + Py_DECREF(slice); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-3] = res; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + PyStackRef_CLOSE(container); + if (res_o == NULL) CEVAL_GOTO(pop_3_error); + res = PyStackRef_FromPyObjectSteal(res_o); } + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR); - PREDICTED(BINARY_SUBSCR); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef container; - _PyStackRef sub; - _PyStackRef res; - // _SPECIALIZE_BINARY_SUBSCR - { - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - assert(frame->stackpointer == NULL); - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_BinarySubscr(container, sub, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(BINARY_SUBSCR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _BINARY_SUBSCR - { - PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); - PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR); + PREDICTED(BINARY_SUBSCR); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef container; + _PyStackRef sub; + _PyStackRef res; + // _SPECIALIZE_BINARY_SUBSCR + { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + assert(frame->stackpointer == NULL); + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_GetItem(container_o, sub_o); + _Py_Specialize_BinarySubscr(container, sub, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - res = PyStackRef_FromPyObjectSteal(res_o); + DISPATCH_SAME_OPARG(); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + OPCODE_DEFERRED_INC(BINARY_SUBSCR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _BINARY_SUBSCR + { + PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); + PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_GetItem(container_o, sub_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); + res = PyStackRef_FromPyObjectSteal(res_o); } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_DICT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef dict_st; - _PyStackRef sub_st; - _PyStackRef res; - /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - dict_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - PyObject *res_o; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef dict_st; + _PyStackRef sub_st; + _PyStackRef res; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + dict_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int rc = PyDict_GetItemRef(dict, sub, &res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (rc == 0) { _PyFrame_SetStackPointer(frame, stack_pointer); - int rc = PyDict_GetItemRef(dict, sub, &res_o); + _PyErr_SetKeyError(sub); stack_pointer = _PyFrame_GetStackPointer(frame); - if (rc == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetKeyError(sub); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - PyStackRef_CLOSE(dict_st); - PyStackRef_CLOSE(sub_st); - if (rc <= 0) CEVAL_GOTO(pop_2_error); - // not found or error - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); } + PyStackRef_CLOSE(dict_st); + PyStackRef_CLOSE(sub_st); + if (rc <= 0) CEVAL_GOTO(pop_2_error); + // not found or error + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef container; - _PyStackRef getitem; - _PyStackRef sub; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); - } - // _BINARY_SUBSCR_CHECK_FUNC - { - container = stack_pointer[-2]; - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR); - assert(PyFunction_Check(getitem_o)); - uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR); - PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); - assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); - getitem = PyStackRef_FromPyObjectNew(getitem_o); - STAT_INC(BINARY_SUBSCR, hit); - } - // _BINARY_SUBSCR_INIT_CALL - { - sub = stack_pointer[-1]; - new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); - new_frame->localsplus[0] = container; - new_frame->localsplus[1] = sub; - frame->return_offset = 2 ; - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef container; + _PyStackRef getitem; + _PyStackRef sub; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef list_st; - _PyStackRef sub_st; - _PyStackRef res; - /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - #ifdef Py_GIL_DISABLED - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); - stack_pointer = _PyFrame_GetStackPointer(frame); - DEOPT_IF(res_o == NULL, BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - #else - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + // _BINARY_SUBSCR_CHECK_FUNC + { + container = stack_pointer[-2]; + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); + DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR); + assert(PyFunction_Check(getitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); + DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR); + PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); + assert(code->co_argcount == 2); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); - PyObject *res_o = PyList_GET_ITEM(list, index); - assert(res_o != NULL); - Py_INCREF(res_o); - #endif - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - PyStackRef_CLOSE(list_st); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + } + // _BINARY_SUBSCR_INIT_CALL + { + sub = stack_pointer[-1]; + new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + new_frame->localsplus[0] = container; + new_frame->localsplus[1] = sub; + frame->return_offset = 2 ; + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef str_st; - _PyStackRef sub_st; - _PyStackRef res; - /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - str_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); - // Specialize for reading an ASCII character from any string: - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - PyStackRef_CLOSE(str_st); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef list_st; + _PyStackRef sub_st; + _PyStackRef res; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyList_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + stack_pointer = _PyFrame_GetStackPointer(frame); + DEOPT_IF(res_o == NULL, BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + #else + DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + Py_INCREF(res_o); + #endif + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(list_st); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef str_st; + _PyStackRef sub_st; + _PyStackRef res; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + str_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + // Specialize for reading an ASCII character from any string: + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(str_st); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef tuple_st; - _PyStackRef sub_st; - _PyStackRef res; - /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - tuple_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - PyObject *res_o = PyTuple_GET_ITEM(tuple, index); - assert(res_o != NULL); - Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - PyStackRef_CLOSE(tuple_st); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef tuple_st; + _PyStackRef sub_st; + _PyStackRef res; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + tuple_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyTuple_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + Py_INCREF(res_o); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(tuple_st); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_LIST); - _PyStackRef *values; - _PyStackRef list; - values = &stack_pointer[-oparg]; - PyObject *list_o = _PyList_FromStackRefSteal(values, oparg); - if (list_o == NULL) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - list = PyStackRef_FromPyObjectSteal(list_o); - stack_pointer[-oparg] = list; - stack_pointer += 1 - oparg; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_LIST); + _PyStackRef *values; + _PyStackRef list; + values = &stack_pointer[-oparg]; + PyObject *list_o = _PyList_FromStackRefSteal(values, oparg); + if (list_o == NULL) { + stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + CEVAL_GOTO(error); } + list = PyStackRef_FromPyObjectSteal(list_o); + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_MAP); - _PyStackRef *values; - _PyStackRef map; - values = &stack_pointer[-oparg*2]; - STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); - if (CONVERSION_FAILED(values_o)) { - for (int _i = oparg*2; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); - } - { - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *map_o = _PyDict_FromItems( - values_o, 2, - values_o+1, 2, - oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_MAP); + _PyStackRef *values; + _PyStackRef map; + values = &stack_pointer[-oparg*2]; + STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); + if (CONVERSION_FAILED(values_o)) { for (int _i = oparg*2; --_i >= 0;) { PyStackRef_CLOSE(values[_i]); } - if (map_o == NULL) { + { stack_pointer += -oparg*2; assert(WITHIN_STACK_BOUNDS()); CEVAL_GOTO(error); } - map = PyStackRef_FromPyObjectSteal(map_o); - stack_pointer[-oparg*2] = map; - stack_pointer += 1 - oparg*2; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *map_o = _PyDict_FromItems( + values_o, 2, + values_o+1, 2, + oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + for (int _i = oparg*2; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + if (map_o == NULL) { + stack_pointer += -oparg*2; assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + CEVAL_GOTO(error); } + map = PyStackRef_FromPyObjectSteal(map_o); + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SET); - _PyStackRef *values; - _PyStackRef set; - values = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *set_o = PySet_New(NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (set_o == NULL) { - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); - } - { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SET); + _PyStackRef *values; + _PyStackRef set; + values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *set_o = PySet_New(NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (set_o == NULL) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); } - int err = 0; - for (int i = 0; i < oparg; i++) { - if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PySet_Add(set_o, PyStackRef_AsPyObjectBorrow(values[i])); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - PyStackRef_CLOSE(values[i]); + { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } - if (err != 0) { - Py_DECREF(set_o); - { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } + } + int err = 0; + for (int i = 0; i < oparg; i++) { + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PySet_Add(set_o, PyStackRef_AsPyObjectBorrow(values[i])); + stack_pointer = _PyFrame_GetStackPointer(frame); } - set = PyStackRef_FromPyObjectSteal(set_o); - stack_pointer[-oparg] = set; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + PyStackRef_CLOSE(values[i]); } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SLICE); - _PyStackRef start; - _PyStackRef stop; - _PyStackRef step = PyStackRef_NULL; - _PyStackRef slice; - if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } - stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; - start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; - PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); - PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); - PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); - PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); - PyStackRef_CLOSE(start); - PyStackRef_CLOSE(stop); - PyStackRef_XCLOSE(step); - if (slice_o == NULL) { - stack_pointer += -2 - ((oparg == 3) ? 1 : 0); + if (err != 0) { + Py_DECREF(set_o); + { + stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); CEVAL_GOTO(error); } - slice = PyStackRef_FromPyObjectSteal(slice_o); - stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; - stack_pointer += -1 - ((oparg == 3) ? 1 : 0); + } + set = PyStackRef_FromPyObjectSteal(set_o); + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SLICE); + _PyStackRef start; + _PyStackRef stop; + _PyStackRef step = PyStackRef_NULL; + _PyStackRef slice; + if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } + stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; + start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; + PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); + PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); + PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); + PyStackRef_CLOSE(start); + PyStackRef_CLOSE(stop); + PyStackRef_XCLOSE(step); + if (slice_o == NULL) { + stack_pointer += -2 - ((oparg == 3) ? 1 : 0); assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + CEVAL_GOTO(error); } + slice = PyStackRef_FromPyObjectSteal(slice_o); + stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; + stack_pointer += -1 - ((oparg == 3) ? 1 : 0); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_STRING); - _PyStackRef *pieces; - _PyStackRef str; - pieces = &stack_pointer[-oparg]; - STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); - if (CONVERSION_FAILED(pieces_o)) { - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(pieces[_i]); - } - { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - } - PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_STRING); + _PyStackRef *pieces; + _PyStackRef str; + pieces = &stack_pointer[-oparg]; + STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); + if (CONVERSION_FAILED(pieces_o)) { for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(pieces[_i]); } - if (str_o == NULL) { + { stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); CEVAL_GOTO(error); } - str = PyStackRef_FromPyObjectSteal(str_o); - stack_pointer[-oparg] = str; - stack_pointer += 1 - oparg; + } + PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(pieces[_i]); + } + if (str_o == NULL) { + stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + CEVAL_GOTO(error); } + str = PyStackRef_FromPyObjectSteal(str_o); + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_TUPLE); - _PyStackRef *values; - _PyStackRef tup; - values = &stack_pointer[-oparg]; - PyObject *tup_o = _PyTuple_FromStackRefSteal(values, oparg); - if (tup_o == NULL) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - tup = PyStackRef_FromPyObjectSteal(tup_o); - stack_pointer[-oparg] = tup; - stack_pointer += 1 - oparg; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TUPLE); + _PyStackRef *values; + _PyStackRef tup; + values = &stack_pointer[-oparg]; + PyObject *tup_o = _PyTuple_FromStackRefSteal(values, oparg); + if (tup_o == NULL) { + stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + CEVAL_GOTO(error); } + tup = PyStackRef_FromPyObjectSteal(tup_o); + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CACHE); - assert(0 && "Executing a cache."); - Py_FatalError("Executing a cache."); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CACHE); + assert(0 && "Executing a cache."); + Py_FatalError("Executing a cache."); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL); - PREDICTED(CALL); - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef *func; - _PyStackRef *maybe_self; - _PyStackRef res; - // _SPECIALIZE_CALL - { - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CALL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD - { - args = &stack_pointer[-oparg]; - func = &stack_pointer[-2 - oparg]; - maybe_self = &stack_pointer[-1 - oparg]; - if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); - PyStackRef_CLOSE(temp); - } + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL); + PREDICTED(CALL); + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef res; + // _SPECIALIZE_CALL + { + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); } - // _DO_CALL - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; + OPCODE_DEFERRED_INC(CALL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD + { + args = &stack_pointer[-oparg]; + func = &stack_pointer[-2 - oparg]; + maybe_self = &stack_pointer[-1 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - // Check if the call can be inlined or not - if (Py_TYPE(callable_o) == &PyFunction_Type && + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); + } + } + // _DO_CALL + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && tstate->interp->eval_frame == NULL && ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - args, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - CEVAL_GOTO(error); - } - frame->return_offset = 4 ; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - } + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, total_args, NULL, frame + ); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(res_o); - } - } + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + CEVAL_GOTO(error); } - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + frame->return_offset = 4 ; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } - if (res_o == NULL) { + { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); CEVAL_GOTO(error); } - res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *null; - _PyStackRef *args; - _PyStackRef *init; - _PyStackRef *self; - _PyInterpreterFrame *init_frame; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_AND_ALLOCATE_OBJECT - { - args = &stack_pointer[-oparg]; - null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - init = &stack_pointer[-2 - oparg]; - self = &stack_pointer[-1 - oparg]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(!PyType_Check(callable_o), CALL); - PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL); - assert(tp->tp_new == PyBaseObject_Type.tp_new); - assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); - assert(tp->tp_alloc == PyType_GenericAlloc); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; - PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); - PyCodeObject *code = (PyCodeObject *)init_func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *self_o = PyType_GenericAlloc(tp, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (self_o == NULL) { - CEVAL_GOTO(error); + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(res_o); + } } - self[0] = PyStackRef_FromPyObjectSteal(self_o); - _PyStackRef temp = callable[0]; - init[0] = PyStackRef_FromPyObjectNew(init_func); - PyStackRef_CLOSE(temp); } - // _CREATE_INIT_FRAME - { - args = &stack_pointer[-oparg]; - self = &stack_pointer[-1 - oparg]; - init = &stack_pointer[-2 - oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); - assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); - assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); - stack_pointer = _PyFrame_GetStackPointer(frame); - /* Push self onto stack of shim */ - shim->localsplus[0] = PyStackRef_DUP(self[0]); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, init[0], NULL, args-1, oparg+1, NULL, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - _PyEval_FrameClearAndPop(tstate, shim); - CEVAL_GOTO(error); - } - init_frame = temp; - frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; - /* Account for pushing the extra frame. - * We don't check recursion depth here, - * as it will be checked after start_frame */ - tstate->py_recursion_remaining--; - } - // _PUSH_FRAME - { - new_frame = init_frame; - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); + CEVAL_GOTO(error); } - DISPATCH(); + res = PyStackRef_FromPyObjectSteal(res_o); } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *null; - _PyStackRef *func; - _PyStackRef *self; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS - { - null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL); - } - // _INIT_CALL_BOUND_METHOD_EXACT_ARGS - { - func = &stack_pointer[-2 - oparg]; - self = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - STAT_INC(CALL, hit); - self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - PyStackRef_CLOSE(temp); - } - // flush - // _CHECK_FUNCTION_VERSION - { - callable = &stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); - } - // _CHECK_FUNCTION_EXACT_ARGS - { - self_or_null = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - assert(PyFunction_Check(callable_o)); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); - } - // _CHECK_STACK_SPACE - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); - } - // _INIT_CALL_PY_EXACT_ARGS - { - args = &stack_pointer[-oparg]; - int has_self = !PyStackRef_IsNull(self_or_null[0]); - STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null[0]; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; - } - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - stack_pointer += -2 - oparg; + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } - DISPATCH(); } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *null; - _PyStackRef *method; - _PyStackRef *self; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_METHOD_VERSION - { - null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL); - PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - } - // _EXPAND_METHOD - { - method = &stack_pointer[-2 - oparg]; - self = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - assert(PyStackRef_IsNull(null[0])); - assert(Py_TYPE(callable_o) == &PyMethod_Type); - self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - _PyStackRef temp = callable[0]; - method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(method[0])); - PyStackRef_CLOSE(temp); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS){ + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef *args; + _PyStackRef *init; + _PyStackRef *self; + _PyInterpreterFrame *init_frame; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_AND_ALLOCATE_OBJECT + { + args = &stack_pointer[-oparg]; + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + init = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL); + PyTypeObject *tp = (PyTypeObject *)callable_o; + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL); + assert(tp->tp_new == PyBaseObject_Type.tp_new); + assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_alloc == PyType_GenericAlloc); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; + PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); + PyCodeObject *code = (PyCodeObject *)init_func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *self_o = PyType_GenericAlloc(tp, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (self_o == NULL) { + CEVAL_GOTO(error); } - // flush - // _PY_FRAME_GENERAL - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - args, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - // The frame has stolen all the arguments from the stack. - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - CEVAL_GOTO(error); - } - new_frame = temp; + self[0] = PyStackRef_FromPyObjectSteal(self_o); + _PyStackRef temp = callable[0]; + init[0] = PyStackRef_FromPyObjectNew(init_func); + PyStackRef_CLOSE(temp); + } + // _CREATE_INIT_FRAME + { + args = &stack_pointer[-oparg]; + self = &stack_pointer[-1 - oparg]; + init = &stack_pointer[-2 - oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); + stack_pointer = _PyFrame_GetStackPointer(frame); + /* Push self onto stack of shim */ + shim->localsplus[0] = PyStackRef_DUP(self[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, init[0], NULL, args-1, oparg+1, NULL, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + _PyEval_FrameClearAndPop(tstate, shim); + CEVAL_GOTO(error); } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif + init_frame = temp; + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; + /* Account for pushing the extra frame. + * We don't check recursion depth here, + * as it will be checked after start_frame */ + tstate->py_recursion_remaining--; + } + // _PUSH_FRAME + { + new_frame = init_frame; + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS){ + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef *func; + _PyStackRef *self; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + { + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL); + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + func = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + STAT_INC(CALL, hit); + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + PyStackRef_CLOSE(temp); + } + // flush + // _CHECK_FUNCTION_VERSION + { + callable = &stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL); + } + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null[0]); + STAT_INC(CALL, hit); + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null[0]; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS){ + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef *method; + _PyStackRef *self; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_METHOD_VERSION + { + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL); + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + DEOPT_IF(!PyFunction_Check(func), CALL); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + } + // _EXPAND_METHOD + { + method = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + assert(PyStackRef_IsNull(null[0])); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable[0]; + method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(method[0])); + PyStackRef_CLOSE(temp); + } + // flush + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; } - DISPATCH(); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + // The frame has stolen all the arguments from the stack. + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + CEVAL_GOTO(error); + } + new_frame = temp; } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_CLASS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_CLASS - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - DEOPT_IF(!PyType_Check(callable_o), CALL); - PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); - STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_CLASS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_CLASS + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + DEOPT_IF(!PyType_Check(callable_o), CALL); + PyTypeObject *tp = (PyTypeObject *)callable_o; + DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + STAT_INC(CALL, hit); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); CEVAL_GOTO(error); } - res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL functions, without keywords */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - /* res = func(self, args, nargs) */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable_o), - args_o, - total_args); + int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST_WITH_KEYWORDS - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); - STAT_INC(CALL, hit); - /* res = func(self, args, nargs, kwnames) */ - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* Builtin METH_FASTCALL functions, without keywords */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + /* res = func(self, args, nargs) */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); CEVAL_GOTO(error); } - res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable_o), + args_o, + total_args); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_O); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_O - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - /* Builtin METH_O functions */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL); - // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - _PyStackRef arg = args[0]; - _Py_EnterRecursiveCallTstateUnchecked(tstate); + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); + int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(arg); - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_FUNCTION_EX); - PREDICTED(CALL_FUNCTION_EX); - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; - _PyStackRef func; - _PyStackRef callargs; - _PyStackRef kwargs_in = PyStackRef_NULL; - _PyStackRef tuple; - _PyStackRef kwargs_out = PyStackRef_NULL; - _PyStackRef func_st; - _PyStackRef callargs_st; - _PyStackRef kwargs_st = PyStackRef_NULL; - _PyStackRef result; - // _MAKE_CALLARGS_A_TUPLE - { - if (oparg & 1) { kwargs_in = stack_pointer[-(oparg & 1)]; } - callargs = stack_pointer[-1 - (oparg & 1)]; - func = stack_pointer[-3 - (oparg & 1)]; - PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); - if (PyTuple_CheckExact(callargs_o)) { - tuple = callargs; - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - CEVAL_GOTO(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (tuple_o == NULL) { - CEVAL_GOTO(error); - } - PyStackRef_CLOSE(callargs); - tuple = PyStackRef_FromPyObjectSteal(tuple_o); - } - kwargs_out = kwargs_in; +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; } - // _DO_CALL_FUNCTION_EX - { - kwargs_st = kwargs_out; - callargs_st = tuple; - func_st = func; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - PyObject *result_o; - assert(!_PyErr_Occurred(tstate)); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - assert(PyTuple_CheckExact(callargs)); - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - stack_pointer[-1 - (oparg & 1)] = callargs_st; - if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - CEVAL_GOTO(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(result_o); - } - } - } - } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -2 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( - tstate, func_st, locals, - nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - // Need to sync the stack since we exit with DISPATCH_INLINED. - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { - CEVAL_GOTO(error); - } - assert( 1 == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); - } - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[-1 - (oparg & 1)] = callargs_st; - if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(callargs_st); - PyStackRef_CLOSE(func_st); - if (result_o == NULL) { - stack_pointer += -3 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); + STAT_INC(CALL, hit); + /* res = func(self, args, nargs, kwnames) */ + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); } - result = PyStackRef_FromPyObjectSteal(result_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-3 - (oparg & 1)] = result; - stack_pointer += -2 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 2 + (oparg & 1); + { + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } } - stack_pointer[-3 - (oparg & 1)] = result; - stack_pointer += -2 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_1); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - assert(oparg <= MAX_INTRINSIC_1); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (res_o == NULL) CEVAL_GOTO(pop_1_error); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); + } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; - DISPATCH(); } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_2); - _PyStackRef value2_st; - _PyStackRef value1_st; - _PyStackRef res; - value1_st = stack_pointer[-1]; - value2_st = stack_pointer[-2]; - assert(oparg <= MAX_INTRINSIC_2); - PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); - PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value2_st); - PyStackRef_CLOSE(value1_st); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ISINSTANCE); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_O + { args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - /* isinstance(o, o2) */ + /* Builtin METH_O functions */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } - DEOPT_IF(total_args != 2, CALL); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL); + DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); STAT_INC(CALL, hit); - _PyStackRef cls_stackref = args[1]; - _PyStackRef inst_stackref = args[0]; + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + _PyStackRef arg = args[0]; + _Py_EnterRecursiveCallTstateUnchecked(tstate); _PyFrame_SetStackPointer(frame, stack_pointer); - int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (retval < 0) { + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(arg); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); CEVAL_GOTO(error); } - res = retval ? PyStackRef_True : PyStackRef_False; - assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(inst_stackref); - PyStackRef_CLOSE(cls_stackref); - PyStackRef_CLOSE(callable[0]); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW); - PREDICTED(CALL_KW); - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef kwnames_in; - _PyStackRef *func; - _PyStackRef *maybe_self; - _PyStackRef kwnames_out; - _PyStackRef res; - // _SPECIALIZE_CALL_KW - { - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CALL_KW); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD_KW - { - kwnames_in = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - func = &stack_pointer[-3 - oparg]; - maybe_self = &stack_pointer[-2 - oparg]; - if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); - PyStackRef_CLOSE(temp); - } - kwnames_out = kwnames_in; +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_FUNCTION_EX); + PREDICTED(CALL_FUNCTION_EX); + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef kwargs_in = PyStackRef_NULL; + _PyStackRef tuple; + _PyStackRef kwargs_out = PyStackRef_NULL; + _PyStackRef func_st; + _PyStackRef callargs_st; + _PyStackRef kwargs_st = PyStackRef_NULL; + _PyStackRef result; + // _MAKE_CALLARGS_A_TUPLE + { + if (oparg & 1) { kwargs_in = stack_pointer[-(oparg & 1)]; } + callargs = stack_pointer[-1 - (oparg & 1)]; + func = stack_pointer[-3 - (oparg & 1)]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (PyTuple_CheckExact(callargs_o)) { + tuple = callargs; } - // _DO_CALL_KW - { - kwnames = kwnames_out; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + CEVAL_GOTO(error); } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - // Check if the call can be inlined or not - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-1] = kwnames; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - args, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(kwnames); - // Sync stack explicitly since we leave using DISPATCH_INLINED(). - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - CEVAL_GOTO(error); - } - assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); - frame->return_offset = 4 ; - DISPATCH_INLINED(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { + CEVAL_GOTO(error); } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - PyStackRef_CLOSE(kwnames); - { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } + PyStackRef_CLOSE(callargs); + tuple = PyStackRef_FromPyObjectSteal(tuple_o); + } + kwargs_out = kwargs_in; + } + // _DO_CALL_FUNCTION_EX + { + kwargs_st = kwargs_out; + callargs_st = tuple; + func_st = func; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-1 - (oparg & 1)] = callargs_st; + if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + CEVAL_GOTO(error); } - stack_pointer[-1] = kwnames; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); + result_o = PyObject_Call(func, callargs, kwargs); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); - if (res_o == NULL) { + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); + frame, this_instr, func, arg); stack_pointer = _PyFrame_GetStackPointer(frame); } else { _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); + frame, this_instr, func, arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - Py_CLEAR(res_o); + Py_CLEAR(result_o); } } } - PyStackRef_CLOSE(kwnames); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - if (res_o == NULL) { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *null; - _PyStackRef kwnames; - _PyStackRef *method; - _PyStackRef *self; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW); } - // _CHECK_METHOD_VERSION_KW - { - null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW); - PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL_KW); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW); - } - // _EXPAND_METHOD_KW - { - method = &stack_pointer[-3 - oparg]; - self = &stack_pointer[-2 - oparg]; - _PyStackRef callable_s = callable[0]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable_s); - assert(PyStackRef_IsNull(null[0])); - assert(Py_TYPE(callable_o) == &PyMethod_Type); - self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(method[0])); - PyStackRef_CLOSE(callable_s); - } - // flush - // _PY_FRAME_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer += -2 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Need to sync the stack since we exit with DISPATCH_INLINED. + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + CEVAL_GOTO(error); + } + assert( 1 == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + stack_pointer[-1 - (oparg & 1)] = callargs_st; + if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - args, positional_args, kwnames_o, frame - ); + result_o = PyObject_Call(func, callargs, kwargs); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(kwnames); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - CEVAL_GOTO(error); - } - new_frame = temp; } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(kwargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(callargs_st); + PyStackRef_CLOSE(func_st); + if (result_o == NULL) { + stack_pointer += -3 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + result = PyStackRef_FromPyObjectSteal(result_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3 - (oparg & 1)] = result; + stack_pointer += -2 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 2 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } - DISPATCH(); } + stack_pointer[-3 - (oparg & 1)] = result; + stack_pointer += -2 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_NON_PY); - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef kwnames; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CHECK_IS_NOT_PY_CALLABLE_KW - { - callable = &stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL_KW); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW); - } - // _CALL_KW_NON_PY - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - #if TIER_ONE - assert(opcode != INSTRUMENTED_CALL); - #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - PyStackRef_CLOSE(kwnames); - { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(kwnames); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 2 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_1); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(oparg <= MAX_INTRINSIC_1); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) CEVAL_GOTO(pop_1_error); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-1] = res; + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_2); + _PyStackRef value2_st; + _PyStackRef value1_st; + _PyStackRef res; + value1_st = stack_pointer[-1]; + value2_st = stack_pointer[-2]; + assert(oparg <= MAX_INTRINSIC_2); + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value2_st); + PyStackRef_CLOSE(value1_st); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ISINSTANCE); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* isinstance(o, o2) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + DEOPT_IF(total_args != 2, CALL); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL); + STAT_INC(CALL, hit); + _PyStackRef cls_stackref = args[1]; + _PyStackRef inst_stackref = args[0]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (retval < 0) { + CEVAL_GOTO(error); } + res = retval ? PyStackRef_True : PyStackRef_False; + assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(inst_stackref); + PyStackRef_CLOSE(cls_stackref); + PyStackRef_CLOSE(callable[0]); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_PY); - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef kwnames; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW); + PREDICTED(CALL_KW); + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef kwnames_in; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef kwnames_out; + _PyStackRef res; + // _SPECIALIZE_CALL_KW + { + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); } - // _CHECK_FUNCTION_VERSION_KW - { - callable = &stack_pointer[-3 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); + OPCODE_DEFERRED_INC(CALL_KW); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + kwnames_in = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + func = &stack_pointer[-3 - oparg]; + maybe_self = &stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL_KW); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); + } + kwnames_out = kwnames_in; + } + // _DO_CALL_KW + { + kwnames = kwnames_out; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; } - // _PY_FRAME_KW + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-1] = kwnames; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable[0], locals, args, positional_args, kwnames_o, frame ); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(kwnames); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. + // Sync stack explicitly since we leave using DISPATCH_INLINED(). stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { CEVAL_GOTO(error); } - new_frame = temp; + assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4 ; + DISPATCH_INLINED(new_frame); } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); + } } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); + stack_pointer[-1] = kwnames; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(res_o); + } + } + } + PyStackRef_CLOSE(kwnames); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); } - DISPATCH(); + if (res_o == NULL) { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); } + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LEN); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - /* len(o) */ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS){ + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef kwnames; + _PyStackRef *method; + _PyStackRef *self; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + } + // _CHECK_METHOD_VERSION_KW + { + null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW); + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + DEOPT_IF(!PyFunction_Check(func), CALL_KW); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW); + } + // _EXPAND_METHOD_KW + { + method = &stack_pointer[-3 - oparg]; + self = &stack_pointer[-2 - oparg]; + _PyStackRef callable_s = callable[0]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable_s); + assert(PyStackRef_IsNull(null[0])); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(method[0])); + PyStackRef_CLOSE(callable_s); + } + // flush + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.len, CALL); - STAT_INC(CALL, hit); - _PyStackRef arg_stackref = args[0]; - PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); - Py_ssize_t len_i = PyObject_Length(arg); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, positional_args, kwnames_o, frame + ); stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) { - CEVAL_GOTO(error); - } - PyObject *res_o = PyLong_FromSsize_t(len_i); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res_o == NULL) { + PyStackRef_CLOSE(kwnames); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { CEVAL_GOTO(error); } - PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(arg_stackref); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + new_frame = temp; } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LIST_APPEND); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self; - _PyStackRef arg; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - arg = stack_pointer[-1]; - self = stack_pointer[-2]; - callable = stack_pointer[-3]; - assert(oparg == 1); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL); - assert(self_o != NULL); - DEOPT_IF(!PyList_Check(self_o), CALL); - DEOPT_IF(!LOCK_OBJECT(self_o), CALL); - STAT_INC(CALL, hit); - int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); - UNLOCK_OBJECT(self_o); - PyStackRef_CLOSE(self); - PyStackRef_CLOSE(callable); - if (err) CEVAL_GOTO(pop_3_error); + // _SAVE_RETURN_OFFSET + { #if TIER_ONE - // Skip the following POP_TOP. This is done here in tier one, and - // during trace projection in tier two: - assert(next_instr->op.code == POP_TOP); - SKIP_OVER(1); + frame->return_offset = (uint16_t)(next_instr - this_instr); #endif - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); - PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFast cfunc = - (PyCFunctionFast)(void(*)(void))meth->ml_meth; - PyObject *res_o = cfunc(self, (args_o + 1), nargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_NON_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef kwnames; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE_KW + { + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(PyFunction_Check(callable_o), CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW); + } + // _CALL_KW_NON_PY + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -2 - oparg; + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + { + stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); CEVAL_GOTO(error); } - res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(kwnames); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); - PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); - STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 2 + oparg; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); } + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_NOARGS - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - _PyStackRef self_stackref = args[0]; - PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); - // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - _Py_EnterRecursiveCallTstateUnchecked(tstate); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(self_stackref); - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef kwnames; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + } + // _CHECK_FUNCTION_VERSION_KW + { + callable = &stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL_KW); + } + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(kwnames); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + if (temp == NULL) { + CEVAL_GOTO(error); + } + new_frame = temp; + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_O - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); - // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); - _PyStackRef arg_stackref = args[1]; - _PyStackRef self_stackref = args[0]; - DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - _Py_EnterRecursiveCallTstateUnchecked(tstate); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, - PyStackRef_AsPyObjectBorrow(self_stackref), - PyStackRef_AsPyObjectBorrow(arg_stackref)); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(self_stackref); - PyStackRef_CLOSE(arg_stackref); - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LEN); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* len(o) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable_o != interp->callable_cache.len, CALL); + STAT_INC(CALL, hit); + _PyStackRef arg_stackref = args[0]; + PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (len_i < 0) { + CEVAL_GOTO(error); + } + PyObject *res_o = PyLong_FromSsize_t(len_i); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + CEVAL_GOTO(error); + } + PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(arg_stackref); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LIST_APPEND); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self; + _PyStackRef arg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + arg = stack_pointer[-1]; + self = stack_pointer[-2]; + callable = stack_pointer[-3]; + assert(oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL); + assert(self_o != NULL); + DEOPT_IF(!PyList_Check(self_o), CALL); + DEOPT_IF(!LOCK_OBJECT(self_o), CALL); + STAT_INC(CALL, hit); + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); + PyStackRef_CLOSE(self); + PyStackRef_CLOSE(callable); + if (err) CEVAL_GOTO(pop_3_error); + #if TIER_ONE + // Skip the following POP_TOP. This is done here in tier one, and + // during trace projection in tier two: + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); + #endif + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 1 + oparg; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + /* Builtin METH_FASTCALL methods, without keywords */ + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + int nargs = total_args - 1; + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFast cfunc = + (PyCFunctionFast)(void(*)(void))meth->ml_meth; + PyObject *res_o = cfunc(self, (args_o + 1), nargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Clear the stack of the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_NON_PY_GENERAL); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CHECK_IS_NOT_PY_CALLABLE - { - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; } - // _CALL_NON_PY_GENERAL - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - #if TIER_ONE - assert(opcode != INSTRUMENTED_CALL); - #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + PyTypeObject *d_type = method->d_common.d_type; + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + STAT_INC(CALL, hit); + int nargs = total_args - 1; + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); } - if (res_o == NULL) { + { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); CEVAL_GOTO(error); } - res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_FUNCTION_VERSION - { - callable = &stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); - } - // _CHECK_FUNCTION_EXACT_ARGS - { - self_or_null = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - assert(PyFunction_Check(callable_o)); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); - } - // _CHECK_STACK_SPACE - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); - } - // _INIT_CALL_PY_EXACT_ARGS - { - args = &stack_pointer[-oparg]; - int has_self = !PyStackRef_IsNull(self_or_null[0]); - STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null[0]; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; - } - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_NOARGS + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + assert(oparg == 0 || oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + DEOPT_IF(total_args != 1, CALL); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } - DISPATCH(); } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_GENERAL); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_O + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; } - // _CHECK_FUNCTION_VERSION - { - callable = &stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_O, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + _PyStackRef arg_stackref = args[1]; + _PyStackRef self_stackref = args[0]; + DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, + PyStackRef_AsPyObjectBorrow(self_stackref), + PyStackRef_AsPyObjectBorrow(arg_stackref)); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(arg_stackref); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } - // _PY_FRAME_GENERAL - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - args, total_args, NULL, frame - ); + int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - // The frame has stolen all the arguments from the stack. - stack_pointer += -2 - oparg; + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_NON_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE + { + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(PyFunction_Check(callable_o), CALL); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL); + } + // _CALL_NON_PY_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_CLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); CEVAL_GOTO(error); } - new_frame = temp; } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); } - DISPATCH(); + res = PyStackRef_FromPyObjectSteal(res_o); } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_STR_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_STR_1 - { - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL); - STAT_INC(CALL, hit); + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Str(arg_o); + int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(arg); - if (res_o == NULL) CEVAL_GOTO(pop_3_error); - res = PyStackRef_FromPyObjectSteal(res_o); + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-3] = res; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS){ + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_FUNCTION_VERSION + { + callable = &stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL); + } + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null[0]); + STAT_INC(CALL, hit); + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null[0]; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } - stack_pointer[-3] = res; - stack_pointer += -2; + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TUPLE_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_TUPLE_1 - { - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL); - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PySequence_Tuple(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(arg); - if (res_o == NULL) CEVAL_GOTO(pop_3_error); - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-3] = res; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - } +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS){ + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_FUNCTION_VERSION + { + callable = &stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL); + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; } - stack_pointer[-3] = res; - stack_pointer += -2; + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + // The frame has stolen all the arguments from the stack. + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + if (temp == NULL) { + CEVAL_GOTO(error); + } + new_frame = temp; + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TYPE_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_STR_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_STR_1 + { arg = stack_pointer[-1]; null = stack_pointer[-2]; callable = stack_pointer[-3]; @@ -3458,5840 +3139,5277 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS){ PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL); + DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL); STAT_INC(CALL, hit); - res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Str(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(arg); - stack_pointer[-3] = res; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + if (res_o == NULL) CEVAL_GOTO(pop_3_error); + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } } + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EG_MATCH); - _PyStackRef exc_value_st; - _PyStackRef match_type_st; - _PyStackRef rest; - _PyStackRef match; - match_type_st = stack_pointer[-1]; - exc_value_st = stack_pointer[-2]; - PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - PyStackRef_CLOSE(exc_value_st); - PyStackRef_CLOSE(match_type_st); - CEVAL_GOTO(pop_2_error); - } - PyObject *match_o = NULL; - PyObject *rest_o = NULL; +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TUPLE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_TUPLE_1 + { + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL); + STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match_o, &rest_o); + PyObject *res_o = PySequence_Tuple(arg_o); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(exc_value_st); - PyStackRef_CLOSE(match_type_st); - if (res < 0) CEVAL_GOTO(pop_2_error); - assert((match_o == NULL) == (rest_o == NULL)); - if (match_o == NULL) CEVAL_GOTO(pop_2_error); - if (!Py_IsNone(match_o)) { + PyStackRef_CLOSE(arg); + if (res_o == NULL) CEVAL_GOTO(pop_3_error); + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyErr_SetHandledException(match_o); + int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); } - rest = PyStackRef_FromPyObjectSteal(rest_o); - match = PyStackRef_FromPyObjectSteal(match_o); - stack_pointer[-2] = rest; - stack_pointer[-1] = match; - DISPATCH(); } + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EXC_MATCH); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyExceptionInstance_Check(left_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyEval_CheckExceptTypeValid(tstate, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - PyStackRef_CLOSE(right); - CEVAL_GOTO(pop_1_error); - } +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TYPE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL); + STAT_INC(CALL, hit); + res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); + PyStackRef_CLOSE(arg); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EG_MATCH); + _PyStackRef exc_value_st; + _PyStackRef match_type_st; + _PyStackRef rest; + _PyStackRef match; + match_type_st = stack_pointer[-1]; + exc_value_st = stack_pointer[-2]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + PyStackRef_CLOSE(exc_value_st); + PyStackRef_CLOSE(match_type_st); + CEVAL_GOTO(pop_2_error); + } + PyObject *match_o = NULL; + PyObject *rest_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, + &match_o, &rest_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(exc_value_st); + PyStackRef_CLOSE(match_type_st); + if (res < 0) CEVAL_GOTO(pop_2_error); + assert((match_o == NULL) == (rest_o == NULL)); + if (match_o == NULL) CEVAL_GOTO(pop_2_error); + if (!Py_IsNone(match_o)) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PyErr_GivenExceptionMatches(left_o, right_o); + PyErr_SetHandledException(match_o); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } + rest = PyStackRef_FromPyObjectSteal(rest_o); + match = PyStackRef_FromPyObjectSteal(match_o); + stack_pointer[-2] = rest; + stack_pointer[-1] = match; + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EXC_MATCH); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyExceptionInstance_Check(left_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptTypeValid(tstate, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { PyStackRef_CLOSE(right); - b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[-1] = b; - DISPATCH(); + CEVAL_GOTO(pop_1_error); } + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyErr_GivenExceptionMatches(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(right); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[-1] = b; + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(CLEANUP_THROW); - _PyStackRef sub_iter_st; - _PyStackRef last_sent_val_st; - _PyStackRef exc_value_st; - _PyStackRef none; - _PyStackRef value; - exc_value_st = stack_pointer[-1]; - last_sent_val_st = stack_pointer[-2]; - sub_iter_st = stack_pointer[-3]; - PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - #ifndef Py_TAIL_CALL_INTERP - assert(throwflag); - #endif - assert(exc_value && PyExceptionInstance_Check(exc_value)); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(CLEANUP_THROW); + _PyStackRef sub_iter_st; + _PyStackRef last_sent_val_st; + _PyStackRef exc_value_st; + _PyStackRef none; + _PyStackRef value; + exc_value_st = stack_pointer[-1]; + last_sent_val_st = stack_pointer[-2]; + sub_iter_st = stack_pointer[-3]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + #ifndef Py_TAIL_CALL_INTERP + assert(throwflag); + #endif + assert(exc_value && PyExceptionInstance_Check(exc_value)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + none = PyStackRef_None; + value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); + PyStackRef_CLOSE(sub_iter_st); + PyStackRef_CLOSE(last_sent_val_st); + PyStackRef_CLOSE(exc_value_st); + } + else { _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); + _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - none = PyStackRef_None; - value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); - PyStackRef_CLOSE(sub_iter_st); - PyStackRef_CLOSE(last_sent_val_st); - PyStackRef_CLOSE(exc_value_st); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); - monitor_reraise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - none = PyStackRef_NULL; - value = PyStackRef_NULL; - CEVAL_GOTO(exception_unwind); - } - stack_pointer[-3] = none; - stack_pointer[-2] = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + none = PyStackRef_NULL; + value = PyStackRef_NULL; + CEVAL_GOTO(exception_unwind); } + stack_pointer[-3] = none; + stack_pointer[-2] = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP); - PREDICTED(COMPARE_OP); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _SPECIALIZE_COMPARE_OP - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(COMPARE_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP); + PREDICTED(COMPARE_OP); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _SPECIALIZE_COMPARE_OP + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CompareOp(left, right, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); } - // _COMPARE_OP - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert((oparg >> 5) <= Py_GE); + OPCODE_DEFERRED_INC(COMPARE_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + // _COMPARE_OP + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert((oparg >> 5) <= Py_GE); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); + if (oparg & 16) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + int res_bool = PyObject_IsTrue(res_o); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - if (oparg & 16) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res_bool = PyObject_IsTrue(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - Py_DECREF(res_o); - if (res_bool < 0) CEVAL_GOTO(error); - res = res_bool ? PyStackRef_True : PyStackRef_False; - } - else { - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - } + Py_DECREF(res_o); + if (res_bool < 0) CEVAL_GOTO(error); + res = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_FLOAT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP); - } - /* Skip 1 cache entry */ - // _COMPARE_OP_FLOAT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left_o); - double dright = PyFloat_AS_DOUBLE(right_o); - // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg - int sign_ish = COMPARISON_BIT(dleft, dright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - // It's always a bool, so we don't care about oparg & 16. - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_FLOAT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP); } + /* Skip 1 cache entry */ + // _COMPARE_OP_FLOAT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); + // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg + int sign_ish = COMPARISON_BIT(dleft, dright); + PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + // It's always a bool, so we don't care about oparg & 16. + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_INT); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_INT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP); - } - /* Skip 1 cache entry */ - // _COMPARE_OP_INT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_INT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_INT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP); + } + /* Skip 1 cache entry */ + // _COMPARE_OP_INT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && _PyLong_DigitCount((PyLongObject *)right_o) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); - // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg - int sign_ish = COMPARISON_BIT(ileft, iright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - // It's always a bool, so we don't care about oparg & 16. - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); + // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg + int sign_ish = COMPARISON_BIT(ileft, iright); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + // It's always a bool, so we don't care about oparg & 16. } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_STR); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_UNICODE - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP); - } - /* Skip 1 cache entry */ - // _COMPARE_OP_STR - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left_o, right_o); - assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(eq == 0 || eq == 1); - assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); - assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; - // It's always a bool, so we don't care about oparg & 16. - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_STR); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_UNICODE + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP); } + /* Skip 1 cache entry */ + // _COMPARE_OP_STR + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left_o, right_o); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; + // It's always a bool, so we don't care about oparg & 16. + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP); - PREDICTED(CONTAINS_OP); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - // _SPECIALIZE_CONTAINS_OP - { - right = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ContainsOp(right, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CONTAINS_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _CONTAINS_OP - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP); + PREDICTED(CONTAINS_OP); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + // _SPECIALIZE_CONTAINS_OP + { + right = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PySequence_Contains(right_o, left_o); + _Py_Specialize_ContainsOp(right, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - if (res < 0) CEVAL_GOTO(pop_2_error); - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + DISPATCH_SAME_OPARG(); } - stack_pointer[-2] = b; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + OPCODE_DEFERRED_INC(CONTAINS_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP_DICT); - static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - /* Skip 1 cache entry */ - right = stack_pointer[-1]; + // _CONTAINS_OP + { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP); - STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PyDict_Contains(right_o, left_o); + int res = PySequence_Contains(right_o, left_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res < 0) CEVAL_GOTO(pop_2_error); b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); } + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_DICT); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + /* Skip 1 cache entry */ + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP); + STAT_INC(CONTAINS_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyDict_Contains(right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) CEVAL_GOTO(pop_2_error); + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP_SET); - static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - /* Skip 1 cache entry */ - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP); - STAT_INC(CONTAINS_OP, hit); - // Note: both set and frozenset use the same seq_contains method! - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PySet_Contains((PySetObject *)right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - if (res < 0) CEVAL_GOTO(pop_2_error); - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_SET); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + /* Skip 1 cache entry */ + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP); + STAT_INC(CONTAINS_OP, hit); + // Note: both set and frozenset use the same seq_contains method! + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PySet_Contains((PySetObject *)right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) CEVAL_GOTO(pop_2_error); + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CONVERT_VALUE); - _PyStackRef value; - _PyStackRef result; - value = stack_pointer[-1]; - conversion_func conv_fn; - assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = _PyEval_ConversionFuncs[oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (result_o == NULL) CEVAL_GOTO(pop_1_error); - result = PyStackRef_FromPyObjectSteal(result_o); - stack_pointer[-1] = result; - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONVERT_VALUE); + _PyStackRef value; + _PyStackRef result; + value = stack_pointer[-1]; + conversion_func conv_fn; + assert(oparg >= FVC_STR && oparg <= FVC_ASCII); + conv_fn = _PyEval_ConversionFuncs[oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (result_o == NULL) CEVAL_GOTO(pop_1_error); + result = PyStackRef_FromPyObjectSteal(result_o); + stack_pointer[-1] = result; + DISPATCH(); } - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY); - _PyStackRef bottom; - _PyStackRef top; - bottom = stack_pointer[-1 - (oparg-1)]; - assert(oparg > 0); - top = PyStackRef_DUP(bottom); - stack_pointer[0] = top; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY); + _PyStackRef bottom; + _PyStackRef top; + bottom = stack_pointer[-1 - (oparg-1)]; + assert(oparg > 0); + top = PyStackRef_DUP(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY_FREE_VARS); - /* Copy closure variables to free variables */ - PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - PyObject *closure = func->func_closure; - assert(oparg == co->co_nfreevars); - int offset = co->co_nlocalsplus - oparg; - for (int i = 0; i < oparg; ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); - } - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY_FREE_VARS); + /* Copy closure variables to free variables */ + PyCodeObject *co = _PyFrame_GetCode(frame); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; + assert(oparg == co->co_nfreevars); + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); + } + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_ATTR); - _PyStackRef owner; - owner = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(owner); - if (err) CEVAL_GOTO(pop_1_error); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_ATTR); + _PyStackRef owner; + owner = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(owner); + if (err) CEVAL_GOTO(pop_1_error); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_DEREF); - PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - // Can't use ERROR_IF here. - // Fortunately we don't need its superpower. - PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); - if (oldobj == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); - } - Py_DECREF(oldobj); - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_DEREF); + PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + // Can't use ERROR_IF here. + // Fortunately we don't need its superpower. + PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); + if (oldobj == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); } + Py_DECREF(oldobj); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_FAST); - _PyStackRef v = GETLOCAL(oparg); - if (PyStackRef_IsNull(v)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); - } - SETLOCAL(oparg, PyStackRef_NULL); - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_FAST); + _PyStackRef v = GETLOCAL(oparg); + if (PyStackRef_IsNull(v)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); } + SETLOCAL(oparg, PyStackRef_NULL); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_GLOBAL); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_GLOBAL); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_Pop(GLOBALS(), name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Can't use ERROR_IF here. + if (err < 0) { + CEVAL_GOTO(error); + } + if (err == 0) { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Pop(GLOBALS(), name, NULL); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); - // Can't use ERROR_IF here. - if (err < 0) { - CEVAL_GOTO(error); - } - if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); - } - DISPATCH(); + CEVAL_GOTO(error); } + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_NAME); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_NAME); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); - } + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_DelItem(ns, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Can't use ERROR_IF here. + if (err != 0) { _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_DelItem(ns, name); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, + name); stack_pointer = _PyFrame_GetStackPointer(frame); - // Can't use ERROR_IF here. - if (err != 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); - } - DISPATCH(); + CEVAL_GOTO(error); } + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_SUBSCR); - _PyStackRef container; - _PyStackRef sub; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - /* del container[sub] */ - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_SUBSCR); + _PyStackRef container; + _PyStackRef sub; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + /* del container[sub] */ + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (err) CEVAL_GOTO(pop_2_error); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (err) CEVAL_GOTO(pop_2_error); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_MERGE); - _PyStackRef callable; - _PyStackRef dict; - _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - callable = stack_pointer[-5 - (oparg - 1)]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_MERGE); + _PyStackRef callable; + _PyStackRef dict; + _PyStackRef update; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_MergeEx(dict_o, update_o, 2); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_MergeEx(dict_o, update_o, 2); + _PyEval_FormatKwargsError(tstate, callable_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatKwargsError(tstate, callable_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(update); - CEVAL_GOTO(pop_1_error); - } PyStackRef_CLOSE(update); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + CEVAL_GOTO(pop_1_error); } + PyStackRef_CLOSE(update); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_UPDATE); - _PyStackRef dict; - _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_UPDATE); + _PyStackRef dict; + _PyStackRef update; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_Update(dict_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Update(dict_o, update_o); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { + if (matches) { _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, + _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(update_o)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - PyStackRef_CLOSE(update); - CEVAL_GOTO(pop_1_error); + stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(update); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + CEVAL_GOTO(pop_1_error); } + PyStackRef_CLOSE(update); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(END_ASYNC_FOR); - _PyStackRef awaitable_st; - _PyStackRef exc_st; - exc_st = stack_pointer[-1]; - awaitable_st = stack_pointer[-2]; - PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); - assert(exc && PyExceptionInstance_Check(exc)); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(END_ASYNC_FOR); + _PyStackRef awaitable_st; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; + awaitable_st = stack_pointer[-2]; + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(exc && PyExceptionInstance_Check(exc)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + PyStackRef_CLOSE(awaitable_st); + PyStackRef_CLOSE(exc_st); + } + else { + Py_INCREF(exc); _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - PyStackRef_CLOSE(awaitable_st); - PyStackRef_CLOSE(exc_st); - } - else { - Py_INCREF(exc); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(exception_unwind); - } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + CEVAL_GOTO(exception_unwind); } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_FOR); - _PyStackRef value; - value = stack_pointer[-1]; - PyStackRef_CLOSE(value); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_FOR); + _PyStackRef value; + value = stack_pointer[-1]; + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_SEND); - _PyStackRef receiver; - _PyStackRef value; - _PyStackRef val; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - (void)receiver; - val = value; - PyStackRef_CLOSE(receiver); - stack_pointer[-2] = val; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_SEND); + _PyStackRef receiver; + _PyStackRef value; + _PyStackRef val; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + (void)receiver; + val = value; + PyStackRef_CLOSE(receiver); + stack_pointer[-2] = val; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS){ - int opcode = ENTER_EXECUTOR; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(ENTER_EXECUTOR); - #ifdef _Py_TIER2 - PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; - assert(executor->vm_data.index == INSTR_OFFSET() - 1); - assert(executor->vm_data.code == code); - assert(executor->vm_data.valid); - assert(tstate->previous_executor == NULL); - /* If the eval breaker is set then stay in tier 1. - * This avoids any potentially infinite loops - * involving _RESUME_CHECK */ - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - opcode = executor->vm_data.opcode; - oparg = (oparg & ~255) | executor->vm_data.oparg; - next_instr = this_instr; - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - } - DISPATCH_GOTO(); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(ENTER_EXECUTOR); + #ifdef _Py_TIER2 + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; + assert(executor->vm_data.index == INSTR_OFFSET() - 1); + assert(executor->vm_data.code == code); + assert(executor->vm_data.valid); + assert(tstate->previous_executor == NULL); + /* If the eval breaker is set then stay in tier 1. + * This avoids any potentially infinite loops + * involving _RESUME_CHECK */ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + opcode = executor->vm_data.opcode; + oparg = (oparg & ~255) | executor->vm_data.oparg; + next_instr = this_instr; + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } - tstate->previous_executor = Py_None; - Py_INCREF(executor); - GOTO_TIER_TWO(executor); - #else - Py_FatalError("ENTER_EXECUTOR is not supported in this build"); - #endif /* _Py_TIER2 */ - DISPATCH(); + DISPATCH_GOTO(); } + tstate->previous_executor = Py_None; + Py_INCREF(executor); + GOTO_TIER_TWO(executor); + #else + Py_FatalError("ENTER_EXECUTOR is not supported in this build"); + #endif /* _Py_TIER2 */ + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXIT_INIT_CHECK); - _PyStackRef should_be_none; - should_be_none = stack_pointer[-1]; - assert(STACK_LEVEL() == 2); - if (!PyStackRef_IsNone(should_be_none)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyErr_Format(PyExc_TypeError, + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXIT_INIT_CHECK); + _PyStackRef should_be_none; + should_be_none = stack_pointer[-1]; + assert(STACK_LEVEL() == 2); + if (!PyStackRef_IsNone(should_be_none)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXTENDED_ARG); - assert(oparg); - opcode = next_instr->op.code; - oparg = oparg << 8 | next_instr->op.arg; - PRE_DISPATCH_GOTO(); - DISPATCH_GOTO(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXTENDED_ARG); + assert(oparg); + opcode = next_instr->op.code; + oparg = oparg << 8 | next_instr->op.arg; + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_SIMPLE); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - /* If value is a unicode object, then we know the result - * of format(value) is value itself. */ - if (!PyUnicode_CheckExact(value_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Format(value_o, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (res_o == NULL) CEVAL_GOTO(pop_1_error); - res = PyStackRef_FromPyObjectSteal(res_o); - } - else { - res = value; - } - stack_pointer[-1] = res; - DISPATCH(); - } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_WITH_SPEC); - _PyStackRef value; - _PyStackRef fmt_spec; - _PyStackRef res; - fmt_spec = stack_pointer[-1]; - value = stack_pointer[-2]; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_SIMPLE); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + /* If value is a unicode object, then we know the result + * of format(value) is value itself. */ + if (!PyUnicode_CheckExact(value_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + PyObject *res_o = PyObject_Format(value_o, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - PyStackRef_CLOSE(fmt_spec); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); + if (res_o == NULL) CEVAL_GOTO(pop_1_error); res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); } + else { + res = value; + } + stack_pointer[-1] = res; + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_WITH_SPEC); + _PyStackRef value; + _PyStackRef fmt_spec; + _PyStackRef res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + PyStackRef_CLOSE(fmt_spec); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER); - PREDICTED(FOR_ITER); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef iter; - _PyStackRef next; - // _SPECIALIZE_FOR_ITER - { - iter = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ForIter(iter, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(FOR_ITER); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ - } - // _FOR_ITER - { - /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER); + PREDICTED(FOR_ITER); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef iter; + _PyStackRef next; + // _SPECIALIZE_FOR_ITER + { + iter = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + _Py_Specialize_ForIter(iter, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_o == NULL) { - if (_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - CEVAL_GOTO(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(FOR_ITER); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + // _FOR_ITER + { + /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_o == NULL) { + if (_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!matches) { + CEVAL_GOTO(error); } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - PyStackRef_CLOSE(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */ - JUMPBY(oparg + 2); - DISPATCH(); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); } - next = PyStackRef_FromPyObjectSteal(next_o); - // Common case: no jump, leave it to the code generator + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + PyStackRef_CLOSE(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */ + JUMPBY(oparg + 2); + DISPATCH(); } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + next = PyStackRef_FromPyObjectSteal(next_o); + // Common case: no jump, leave it to the code generator } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_GEN); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyInterpreterFrame *gen_frame; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); - } - // _FOR_ITER_GEN_FRAME - { - iter = stack_pointer[-1]; - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); - STAT_INC(FOR_ITER, hit); - gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, PyStackRef_None); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - gen_frame->previous = frame; - // oparg is the return offset from the next instruction. - frame->return_offset = (uint16_t)( 2 + oparg); - } - // _PUSH_FRAME - { - new_frame = gen_frame; - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_GEN); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyInterpreterFrame *gen_frame; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); + } + // _FOR_ITER_GEN_FRAME + { + iter = stack_pointer[-1]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); + STAT_INC(FOR_ITER, hit); + gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, PyStackRef_None); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + gen_frame->previous = frame; + // oparg is the return offset from the next instruction. + frame->return_offset = (uint16_t)( 2 + oparg); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_LIST); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_LIST - { - iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER); - } - // _ITER_JUMP_LIST - { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - STAT_INC(FOR_ITER, hit); - PyListObject *seq = it->it_seq; - if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { - it->it_index = -1; - #ifndef Py_GIL_DISABLED - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); - } - #endif - PyStackRef_CLOSE(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ - JUMPBY(oparg + 2); - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_LIST); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_LIST + { + iter = stack_pointer[-1]; + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER); + } + // _ITER_JUMP_LIST + { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); + STAT_INC(FOR_ITER, hit); + PyListObject *seq = it->it_seq; + if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { + it->it_index = -1; + #ifndef Py_GIL_DISABLED + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); } + #endif + PyStackRef_CLOSE(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ + JUMPBY(oparg + 2); + DISPATCH(); } - // _ITER_NEXT_LIST - { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyList_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); - } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); } + // _ITER_NEXT_LIST + { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); + PyListObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyList_GET_SIZE(seq)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_RANGE); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_RANGE - { - iter = stack_pointer[-1]; - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); - } - // _ITER_JUMP_RANGE - { - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - STAT_INC(FOR_ITER, hit); - if (r->len <= 0) { - STACK_SHRINK(1); - PyStackRef_CLOSE(iter); - // Jump over END_FOR and POP_TOP instructions. - JUMPBY(oparg + 2); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_RANGE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_RANGE + { + iter = stack_pointer[-1]; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + } + // _ITER_JUMP_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + STAT_INC(FOR_ITER, hit); + if (r->len <= 0) { + STACK_SHRINK(1); + PyStackRef_CLOSE(iter); + // Jump over END_FOR and POP_TOP instructions. + JUMPBY(oparg + 2); + DISPATCH(); } - // _ITER_NEXT_RANGE - { - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - assert(r->len > 0); - long value = r->start; - r->start = value + r->step; - r->len--; - PyObject *res = PyLong_FromLong(value); - if (res == NULL) CEVAL_GOTO(error); - next = PyStackRef_FromPyObjectSteal(res); - } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); } + // _ITER_NEXT_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + assert(r->len > 0); + long value = r->start; + r->start = value + r->step; + r->len--; + PyObject *res = PyLong_FromLong(value); + if (res == NULL) CEVAL_GOTO(error); + next = PyStackRef_FromPyObjectSteal(res); + } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_TUPLE); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_TUPLE - { - iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER); - } - // _ITER_JUMP_TUPLE - { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - STAT_INC(FOR_ITER, hit); - PyTupleObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); - } - PyStackRef_CLOSE(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ - JUMPBY(oparg + 2); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_TUPLE + { + iter = stack_pointer[-1]; + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER); + } + // _ITER_JUMP_TUPLE + { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); + STAT_INC(FOR_ITER, hit); + PyTupleObject *seq = it->it_seq; + if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); + } + PyStackRef_CLOSE(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ + JUMPBY(oparg + 2); + DISPATCH(); } - // _ITER_NEXT_TUPLE - { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - PyTupleObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); - } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); } + // _ITER_NEXT_TUPLE + { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); + PyTupleObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyTuple_GET_SIZE(seq)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); + } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AITER); - _PyStackRef obj; - _PyStackRef iter; - obj = stack_pointer[-1]; - unaryfunc getter = NULL; - PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); - PyObject *iter_o; - PyTypeObject *type = Py_TYPE(obj_o); - if (type->tp_as_async != NULL) { - getter = type->tp_as_async->am_aiter; - } - if (getter == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AITER); + _PyStackRef obj; + _PyStackRef iter; + obj = stack_pointer[-1]; + unaryfunc getter = NULL; + PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); + PyObject *iter_o; + PyTypeObject *type = Py_TYPE(obj_o); + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; + } + if (getter == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(obj); - CEVAL_GOTO(pop_1_error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - iter_o = (*getter)(obj_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); - if (iter_o == NULL) CEVAL_GOTO(pop_1_error); - if (Py_TYPE(iter_o)->tp_as_async == NULL || + CEVAL_GOTO(pop_1_error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + iter_o = (*getter)(obj_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(obj); + if (iter_o == NULL) CEVAL_GOTO(pop_1_error); + if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter_o)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - Py_DECREF(iter_o); - CEVAL_GOTO(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; - DISPATCH(); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(iter_o); + CEVAL_GOTO(error); } + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[-1] = iter; + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ANEXT); - _PyStackRef aiter; - _PyStackRef awaitable; - aiter = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (awaitable_o == NULL) { - CEVAL_GOTO(error); - } - awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); - stack_pointer[0] = awaitable; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ANEXT); + _PyStackRef aiter; + _PyStackRef awaitable; + aiter = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (awaitable_o == NULL) { + CEVAL_GOTO(error); } + awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); + stack_pointer[0] = awaitable; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AWAITABLE); - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(iterable); - if (iter_o == NULL) CEVAL_GOTO(pop_1_error); - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AWAITABLE); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(iterable); + if (iter_o == NULL) CEVAL_GOTO(pop_1_error); + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[-1] = iter; + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ITER); - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - /* before: [obj]; after [getiter(obj)] */ - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(iterable); - if (iter_o == NULL) CEVAL_GOTO(pop_1_error); - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ITER); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + /* before: [obj]; after [getiter(obj)] */ + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(iterable); + if (iter_o == NULL) CEVAL_GOTO(pop_1_error); + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[-1] = iter; + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_LEN); - _PyStackRef obj; - _PyStackRef len; - obj = stack_pointer[-1]; - // PUSH(len(TOS)) - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) CEVAL_GOTO(error); - PyObject *len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) CEVAL_GOTO(error); - len = PyStackRef_FromPyObjectSteal(len_o); - stack_pointer[0] = len; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_LEN); + _PyStackRef obj; + _PyStackRef len; + obj = stack_pointer[-1]; + // PUSH(len(TOS)) + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (len_i < 0) CEVAL_GOTO(error); + PyObject *len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) CEVAL_GOTO(error); + len = PyStackRef_FromPyObjectSteal(len_o); + stack_pointer[0] = len; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_YIELD_FROM_ITER); - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - /* before: [obj]; after [getiter(obj)] */ - PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); - if (PyCoro_CheckExact(iterable_o)) { - /* `iterable` is a coroutine */ - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { - /* and it is used in a 'yield from' expression of a - regular generator. */ - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_TypeError, + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_YIELD_FROM_ITER); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + /* before: [obj]; after [getiter(obj)] */ + PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); + if (PyCoro_CheckExact(iterable_o)) { + /* `iterable` is a coroutine */ + if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + /* and it is used in a 'yield from' expression of a + regular generator. */ + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); - } + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); + } + iter = iterable; + } + else { + if (PyGen_CheckExact(iterable_o)) { iter = iterable; } else { - if (PyGen_CheckExact(iterable_o)) { - iter = iterable; - } - else { - /* `iterable` is not a generator. */ - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(iterable_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - CEVAL_GOTO(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - PyStackRef_CLOSE(iterable); + /* `iterable` is not a generator. */ + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(iterable_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + CEVAL_GOTO(error); } + iter = PyStackRef_FromPyObjectSteal(iter_o); + PyStackRef_CLOSE(iterable); } - stack_pointer[-1] = iter; - DISPATCH(); } + stack_pointer[-1] = iter; + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_FROM); - _PyStackRef from; - _PyStackRef res; - from = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) CEVAL_GOTO(error); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_FROM); + _PyStackRef from; + _PyStackRef res; + from = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) CEVAL_GOTO(error); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_NAME); - _PyStackRef level; - _PyStackRef fromlist; - _PyStackRef res; - fromlist = stack_pointer[-1]; - level = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportName(tstate, frame, name, - PyStackRef_AsPyObjectBorrow(fromlist), - PyStackRef_AsPyObjectBorrow(level)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(level); - PyStackRef_CLOSE(fromlist); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_NAME); + _PyStackRef level; + _PyStackRef fromlist; + _PyStackRef res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportName(tstate, frame, name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(level); + PyStackRef_CLOSE(fromlist); + if (res_o == NULL) CEVAL_GOTO(pop_2_error); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS){ - int opcode = INSTRUMENTED_CALL; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef *func; - _PyStackRef *maybe_self; - _PyStackRef res; - /* Skip 3 cache entries */ - // _MAYBE_EXPAND_METHOD - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - func = &stack_pointer[-2 - oparg]; - maybe_self = &stack_pointer[-1 - oparg]; - if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); - PyStackRef_CLOSE(temp); - } + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef res; + /* Skip 3 cache entries */ + // _MAYBE_EXPAND_METHOD + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + func = &stack_pointer[-2 - oparg]; + maybe_self = &stack_pointer[-1 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); } - // _MONITOR_CALL - { - args = &stack_pointer[-oparg]; - maybe_self = &stack_pointer[-1 - oparg]; - func = &stack_pointer[-2 - oparg]; - int is_meth = !PyStackRef_IsNull(maybe_self[0]); - PyObject *function = PyStackRef_AsPyObjectBorrow(func[0]); - PyObject *arg0; - if (is_meth) { - arg0 = PyStackRef_AsPyObjectBorrow(maybe_self[0]); + } + // _MONITOR_CALL + { + args = &stack_pointer[-oparg]; + maybe_self = &stack_pointer[-1 - oparg]; + func = &stack_pointer[-2 - oparg]; + int is_meth = !PyStackRef_IsNull(maybe_self[0]); + PyObject *function = PyStackRef_AsPyObjectBorrow(func[0]); + PyObject *arg0; + if (is_meth) { + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self[0]); + } + else { + if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); } else { - if (oparg) { - arg0 = PyStackRef_AsPyObjectBorrow(args[0]); - } - else { - arg0 = &_PyInstrumentation_MISSING; - } + arg0 = &_PyInstrumentation_MISSING; } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg0 - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) CEVAL_GOTO(error); } - // _DO_CALL - { - self_or_null = maybe_self; - callable = func; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - // Check if the call can be inlined or not - if (Py_TYPE(callable_o) == &PyFunction_Type && + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg0 + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) CEVAL_GOTO(error); + } + // _DO_CALL + { + self_or_null = maybe_self; + callable = func; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && tstate->interp->eval_frame == NULL && ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - args, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - CEVAL_GOTO(error); - } - frame->return_offset = 4 ; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - } + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, total_args, NULL, frame + ); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(res_o); - } - } + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + CEVAL_GOTO(error); } - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + frame->return_offset = 4 ; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } - if (res_o == NULL) { + { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); CEVAL_GOTO(error); } - res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(res_o); + } } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CEVAL_GOTO(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS){ - int opcode = INSTRUMENTED_CALL_FUNCTION_EX; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); + GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS){ - int opcode = INSTRUMENTED_CALL_KW; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - uint32_t version = read_u32(&this_instr[2].cache); - (void)version; - int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2)); - int total_args = oparg + is_meth; - PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3)); - PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING - : PyStackRef_AsPyObjectBorrow(PEEK(total_args + 1)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) CEVAL_GOTO(error); - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(CALL_KW); - } + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + uint32_t version = read_u32(&this_instr[2].cache); + (void)version; + int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2)); + int total_args = oparg + is_meth; + PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3)); + PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING +: PyStackRef_AsPyObjectBorrow(PEEK(total_args + 1)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) CEVAL_GOTO(error); + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + GO_TO_INSTRUCTION(CALL_KW); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS){ - int opcode = INSTRUMENTED_END_FOR; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_FOR); - _PyStackRef receiver; - _PyStackRef value; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - /* Need to create a fake StopIteration error here, - * to conform to PEP 380 */ - if (PyStackRef_GenCheck(receiver)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - CEVAL_GOTO(error); - } + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_FOR); + _PyStackRef receiver; + _PyStackRef value; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + /* Need to create a fake StopIteration error here, + * to conform to PEP 380 */ + if (PyStackRef_GenCheck(receiver)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + CEVAL_GOTO(error); } - PyStackRef_CLOSE(value); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); } + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS){ - int opcode = INSTRUMENTED_END_SEND; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_SEND); - _PyStackRef receiver; - _PyStackRef value; - _PyStackRef val; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - CEVAL_GOTO(error); - } + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_SEND); + _PyStackRef receiver; + _PyStackRef value; + _PyStackRef val; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + CEVAL_GOTO(error); } - val = value; - PyStackRef_CLOSE(receiver); - stack_pointer[-2] = val; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); } + val = value; + PyStackRef_CLOSE(receiver); + stack_pointer[-2] = val; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS){ - int opcode = INSTRUMENTED_FOR_ITER; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); - /* Skip 1 cache entry */ - _PyStackRef iter_stackref = TOP(); - PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next != NULL) { - PUSH(PyStackRef_FromPyObjectSteal(next)); - } - else { - if (_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - CEVAL_GOTO(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); + /* Skip 1 cache entry */ + _PyStackRef iter_stackref = TOP(); + PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next != NULL) { + PUSH(PyStackRef_FromPyObjectSteal(next)); + } + else { + if (_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!matches) { + CEVAL_GOTO(error); } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - STACK_SHRINK(1); - PyStackRef_CLOSE(iter_stackref); - /* Skip END_FOR and POP_TOP */ - _Py_CODEUNIT *target = next_instr + oparg + 2; - INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH_RIGHT); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); } - DISPATCH(); + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + STACK_SHRINK(1); + PyStackRef_CLOSE(iter_stackref); + /* Skip END_FOR and POP_TOP */ + _Py_CODEUNIT *target = next_instr + oparg + 2; + INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH_RIGHT); } + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS){ - int opcode = INSTRUMENTED_INSTRUCTION; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); - _PyFrame_SetStackPointer(frame, stack_pointer); - int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_opcode < 0) CEVAL_GOTO(error); - next_instr = this_instr; - if (_PyOpcode_Caches[next_opcode]) { - PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); - } - assert(next_opcode > 0 && next_opcode < 256); - opcode = next_opcode; - DISPATCH_GOTO(); - } + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + _PyFrame_SetStackPointer(frame, stack_pointer); + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_opcode < 0) CEVAL_GOTO(error); + next_instr = this_instr; + if (_PyOpcode_Caches[next_opcode]) { + PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); + } + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS){ - int opcode = INSTRUMENTED_JUMP_BACKWARD; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); - /* Skip 1 cache entry */ - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - } - } - // _MONITOR_JUMP_BACKWARD - { - INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); } - DISPATCH(); } + // _MONITOR_JUMP_BACKWARD + { + INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); + } + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS){ - int opcode = INSTRUMENTED_JUMP_FORWARD; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); - DISPATCH(); - } + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS){ - int opcode = INSTRUMENTED_LINE; - (void)(opcode); - { - _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_LINE); - int original_opcode = 0; - if (tstate->tracing) { - PyCodeObject *code = _PyFrame_GetCode(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - original_opcode = code->_co_monitoring->lines[(int)(this_instr - _PyFrame_GetBytecode(frame))].original_opcode; - stack_pointer = _PyFrame_GetStackPointer(frame); - next_instr = this_instr; - } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - original_opcode = _Py_call_instrumentation_line( - tstate, frame, this_instr, prev_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (original_opcode < 0) { - next_instr = this_instr+1; - CEVAL_GOTO(error); - } - next_instr = frame->instr_ptr; - if (next_instr != this_instr) { - DISPATCH(); - } + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_LINE); + int original_opcode = 0; + if (tstate->tracing) { + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = code->_co_monitoring->lines[(int)(this_instr - _PyFrame_GetBytecode(frame))].original_opcode; + stack_pointer = _PyFrame_GetStackPointer(frame); + next_instr = this_instr; + } else { + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = _Py_call_instrumentation_line( + tstate, frame, this_instr, prev_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = this_instr+1; + CEVAL_GOTO(error); } - if (_PyOpcode_Caches[original_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - /* Prevent the underlying instruction from specializing - * and overwriting the instrumentation. */ - PAUSE_ADAPTIVE_COUNTER(cache->counter); + next_instr = frame->instr_ptr; + if (next_instr != this_instr) { + DISPATCH(); } - opcode = original_opcode; - DISPATCH_GOTO(); } + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + /* Prevent the underlying instruction from specializing + * and overwriting the instrumentation. */ + PAUSE_ADAPTIVE_COUNTER(cache->counter); + } + opcode = original_opcode; + DISPATCH_GOTO(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS){ - int opcode = INSTRUMENTED_LOAD_SUPER_ATTR; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); - /* Skip 1 cache entry */ - // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we - // don't want to specialize instrumented instructions - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); - } + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); + /* Skip 1 cache entry */ + // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we + // don't want to specialize instrumented instructions + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS){ - int opcode = INSTRUMENTED_NOT_TAKEN; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); - INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); - DISPATCH(); - } + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS){ - int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); - /* Skip 1 cache entry */ - _PyStackRef cond = POP(); - assert(PyStackRef_BoolCheck(cond)); - int jump = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - DISPATCH(); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); + /* Skip 1 cache entry */ + _PyStackRef cond = POP(); + assert(PyStackRef_BoolCheck(cond)); + int jump = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS){ - int opcode = INSTRUMENTED_POP_JUMP_IF_NONE; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); - /* Skip 1 cache entry */ - _PyStackRef value_stackref = POP(); - int jump = PyStackRef_IsNone(value_stackref); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - else { - PyStackRef_CLOSE(value_stackref); - } - DISPATCH(); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); + /* Skip 1 cache entry */ + _PyStackRef value_stackref = POP(); + int jump = PyStackRef_IsNone(value_stackref); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + else { + PyStackRef_CLOSE(value_stackref); } + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS){ - int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); - /* Skip 1 cache entry */ - _PyStackRef value_stackref = POP(); - int jump = !PyStackRef_IsNone(value_stackref); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - PyStackRef_CLOSE(value_stackref); - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - DISPATCH(); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); + /* Skip 1 cache entry */ + _PyStackRef value_stackref = POP(); + int jump = !PyStackRef_IsNone(value_stackref); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + PyStackRef_CLOSE(value_stackref); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS){ - int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); - /* Skip 1 cache entry */ - _PyStackRef cond = POP(); - assert(PyStackRef_BoolCheck(cond)); - int jump = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - DISPATCH(); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); + /* Skip 1 cache entry */ + _PyStackRef cond = POP(); + assert(PyStackRef_BoolCheck(cond)); + int jump = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS){ - int opcode = INSTRUMENTED_RESUME; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RESUME); - // _LOAD_BYTECODE - { - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RESUME); + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (bytecode == NULL) CEVAL_GOTO(error); + _PyFrame_SetStackPointer(frame, stack_pointer); + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + // Make sure this_instr gets reset correctley for any uops that + // follow + next_instr = frame->instr_ptr; + DISPATCH(); + } + #endif + } + // _MAYBE_INSTRUMENT + { + if (tstate->tracing == 0) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) CEVAL_GOTO(error); - _PyFrame_SetStackPointer(frame, stack_pointer); - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); stack_pointer = _PyFrame_GetStackPointer(frame); - frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; - frame->instr_ptr = bytecode + off; - // Make sure this_instr gets reset correctley for any uops that - // follow - next_instr = frame->instr_ptr; - DISPATCH(); - } - #endif - } - // _MAYBE_INSTRUMENT - { - if (tstate->tracing == 0) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - CEVAL_GOTO(error); - } - next_instr = this_instr; - DISPATCH(); + if (err) { + CEVAL_GOTO(error); } + next_instr = this_instr; + DISPATCH(); } } - // _CHECK_PERIODIC_IF_NOT_YIELD_FROM - { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - } + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); } } - // _MONITOR_RESUME - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) CEVAL_GOTO(error); - if (frame->instr_ptr != this_instr) { - /* Instrumentation has jumped */ - next_instr = frame->instr_ptr; - } + } + // _MONITOR_RESUME + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation( + tstate, oparg > 0, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) CEVAL_GOTO(error); + if (frame->instr_ptr != this_instr) { + /* Instrumentation has jumped */ + next_instr = frame->instr_ptr; } - DISPATCH(); } + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS){ - int opcode = INSTRUMENTED_RETURN_VALUE; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); - _PyStackRef val; - _PyStackRef retval; - _PyStackRef res; - // _RETURN_VALUE_EVENT - { - val = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) CEVAL_GOTO(error); - } - // _RETURN_VALUE - { - retval = val; - #if TIER_ONE - assert(frame != entry_frame); - #endif - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(EMPTY()); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(frame->return_offset); - res = temp; - LLTRACE_RESUME_FRAME(); - } - stack_pointer[0] = res; - stack_pointer += 1; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); + _PyStackRef val; + _PyStackRef retval; + _PyStackRef res; + // _RETURN_VALUE_EVENT + { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) CEVAL_GOTO(error); + } + // _RETURN_VALUE + { + retval = val; + #if TIER_ONE + assert(frame != entry_frame); + #endif + _PyStackRef temp = retval; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(frame->return_offset); + res = temp; + LLTRACE_RESUME_FRAME(); } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS){ - int opcode = INSTRUMENTED_YIELD_VALUE; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); - _PyStackRef val; - _PyStackRef retval; - _PyStackRef value; - // _YIELD_VALUE_EVENT - { - val = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - CEVAL_GOTO(error); - } - if (frame->instr_ptr != this_instr) { - next_instr = frame->instr_ptr; - DISPATCH(); - } + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + _PyStackRef val; + _PyStackRef retval; + _PyStackRef value; + // _YIELD_VALUE_EVENT + { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + CEVAL_GOTO(error); } - // _YIELD_VALUE - { - retval = val; - // NOTE: It's important that YIELD_VALUE never raises an exception! - // The compiler treats any exception raised here as a failed close() - // or throw() call. - #if TIER_ONE - assert(frame != entry_frame); - #endif - frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + if (frame->instr_ptr != this_instr) { + next_instr = frame->instr_ptr; + DISPATCH(); + } + } + // _YIELD_VALUE + { + retval = val; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + #if TIER_ONE + assert(frame != entry_frame); + #endif + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - value = temp; - LLTRACE_RESUME_FRAME(); - } - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + value = temp; + LLTRACE_RESUME_FRAME(); } + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INTERPRETER_EXIT); - _PyStackRef retval; - retval = stack_pointer[-1]; - assert(frame == entry_frame); - assert(_PyFrame_IsIncomplete(frame)); - /* Restore previous frame and return. */ - tstate->current_frame = frame->previous; - assert(!_PyErr_Occurred(tstate)); - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - PyObject *result = PyStackRef_AsPyObjectSteal(retval); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - /* Not strictly necessary, but prevents warnings */ - return result; - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INTERPRETER_EXIT); + _PyStackRef retval; + retval = stack_pointer[-1]; + assert(frame == entry_frame); + assert(_PyFrame_IsIncomplete(frame)); + /* Restore previous frame and return. */ + tstate->current_frame = frame->previous; + assert(!_PyErr_Occurred(tstate)); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + PyObject *result = PyStackRef_AsPyObjectSteal(retval); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + /* Not strictly necessary, but prevents warnings */ + return result; } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IS_OP); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IS_OP); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD); - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - } + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD); + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(error); } - // _JUMP_BACKWARD - { - uint16_t the_counter = read_u16(&this_instr[1].cache); - (void)the_counter; - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); - #ifdef _Py_TIER2 - #if ENABLE_SPECIALIZATION - _Py_BackoffCounter counter = this_instr[1].counter; - if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) { - _Py_CODEUNIT *start = this_instr; - /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ - while (oparg > 255) { - oparg >>= 8; - start--; - } - _PyExecutorObject *executor; - _PyFrame_SetStackPointer(frame, stack_pointer); - int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (optimized <= 0) { - this_instr[1].counter = restart_backoff_counter(counter); - if (optimized < 0) CEVAL_GOTO(error); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - this_instr[1].counter = initial_jump_backoff_counter(); - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(tstate->previous_executor == NULL); - tstate->previous_executor = Py_None; - GOTO_TIER_TWO(executor); - } + } + // _JUMP_BACKWARD + { + uint16_t the_counter = read_u16(&this_instr[1].cache); + (void)the_counter; + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + #ifdef _Py_TIER2 + #if ENABLE_SPECIALIZATION + _Py_BackoffCounter counter = this_instr[1].counter; + if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) { + _Py_CODEUNIT *start = this_instr; + /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ + while (oparg > 255) { + oparg >>= 8; + start--; + } + _PyExecutorObject *executor; + _PyFrame_SetStackPointer(frame, stack_pointer); + int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (optimized <= 0) { + this_instr[1].counter = restart_backoff_counter(counter); + if (optimized < 0) CEVAL_GOTO(error); } else { - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + _PyFrame_SetStackPointer(frame, stack_pointer); + this_instr[1].counter = initial_jump_backoff_counter(); + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(tstate->previous_executor == NULL); + tstate->previous_executor = Py_None; + GOTO_TIER_TWO(executor); } - #endif /* ENABLE_SPECIALIZATION */ - #endif /* _Py_TIER2 */ } - DISPATCH(); + else { + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + #endif /* ENABLE_SPECIALIZATION */ + #endif /* _Py_TIER2 */ } + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); - /* This bytecode is used in the `yield from` or `await` loop. - * If there is an interrupt, we want it handled in the innermost - * generator or coroutine, so we deliberately do not check it here. - * (see bpo-30039). - */ - JUMPBY(-oparg); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + JUMPBY(-oparg); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_FORWARD); - JUMPBY(oparg); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_FORWARD); + JUMPBY(oparg); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_APPEND); - _PyStackRef list; - _PyStackRef v; - v = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), - PyStackRef_AsPyObjectSteal(v)); - if (err < 0) CEVAL_GOTO(pop_1_error); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_APPEND); + _PyStackRef list; + _PyStackRef v; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)); + if (err < 0) CEVAL_GOTO(pop_1_error); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_EXTEND); - _PyStackRef list_st; - _PyStackRef iterable_st; - iterable_st = stack_pointer[-1]; - list_st = stack_pointer[-2 - (oparg-1)]; - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_EXTEND); + _PyStackRef list_st; + _PyStackRef iterable_st; + iterable_st = stack_pointer[-1]; + list_st = stack_pointer[-2 - (oparg-1)]; + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (none_val == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); stack_pointer = _PyFrame_GetStackPointer(frame); - if (none_val == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches && + if (matches && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) - { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Clear(tstate); - _PyErr_Format(tstate, PyExc_TypeError, + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - PyStackRef_CLOSE(iterable_st); - CEVAL_GOTO(pop_1_error); + stack_pointer = _PyFrame_GetStackPointer(frame); } - assert(Py_IsNone(none_val)); PyStackRef_CLOSE(iterable_st); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + CEVAL_GOTO(pop_1_error); } + assert(Py_IsNone(none_val)); + PyStackRef_CLOSE(iterable_st); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR); - PREDICTED(LOAD_ATTR); - _Py_CODEUNIT* const this_instr = next_instr - 10; - (void)this_instr; - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self_or_null = PyStackRef_NULL; - // _SPECIALIZE_LOAD_ATTR - { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR); + PREDICTED(LOAD_ATTR); + _Py_CODEUNIT* const this_instr = next_instr - 10; + (void)this_instr; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self_or_null = PyStackRef_NULL; + // _SPECIALIZE_LOAD_ATTR + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); } - /* Skip 8 cache entries */ - // _LOAD_ATTR - { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - PyObject *attr_o; - if (oparg & 1) { - /* Designed to work in tandem with CALL, pushes two values. */ - attr_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (is_meth) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - meth | self | arg1 | ... | argN - */ - assert(attr_o != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership - } - else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - meth | NULL | arg1 | ... | argN - */ - PyStackRef_CLOSE(owner); - if (attr_o == NULL) CEVAL_GOTO(pop_1_error); - self_or_null = PyStackRef_NULL; - } + OPCODE_DEFERRED_INC(LOAD_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 8 cache entries */ + // _LOAD_ATTR + { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + PyObject *attr_o; + if (oparg & 1) { + /* Designed to work in tandem with CALL, pushes two values. */ + attr_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (is_meth) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr_o != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership } else { - /* Classic, pushes one value. */ - _PyFrame_SetStackPointer(frame, stack_pointer); - attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + meth | NULL | arg1 | ... | argN + */ PyStackRef_CLOSE(owner); if (attr_o == NULL) CEVAL_GOTO(pop_1_error); - /* We need to define self_or_null on all paths */ self_or_null = PyStackRef_NULL; } - attr = PyStackRef_FromPyObjectSteal(attr_o); } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = self_or_null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _CHECK_ATTR_CLASS - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); - assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_CLASS - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = PyStackRef_FromPyObjectNew(descr); - null = PyStackRef_NULL; + else { + /* Classic, pushes one value. */ + _PyFrame_SetStackPointer(frame, stack_pointer); + attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); + if (attr_o == NULL) CEVAL_GOTO(pop_1_error); + /* We need to define self_or_null on all paths */ + self_or_null = PyStackRef_NULL; } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + attr = PyStackRef_FromPyObjectSteal(attr_o); } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _CHECK_ATTR_CLASS - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); - assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); - } - // _GUARD_TYPE_VERSION - { - uint32_t type_version = read_u32(&this_instr[4].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - // _LOAD_ATTR_CLASS - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = PyStackRef_FromPyObjectNew(descr); - null = PyStackRef_NULL; - PyStackRef_CLOSE(owner); - } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS){ + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + assert(type_version != 0); + DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - /* Skip 1 cache entry */ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS){ + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + { owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *getattribute = read_obj(&this_instr[6].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner_o); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); - assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)getattribute; - assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); + } + // _GUARD_TYPE_VERSION + { + uint32_t type_version = read_u32(&this_instr[4].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( - tstate, PyStackRef_FromPyObjectNew(f), 2, frame); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); - new_frame->localsplus[0] = owner; - new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); - frame->return_offset = 10 ; - DISPATCH_INLINED(new_frame); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS){ + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + /* Skip 1 cache entry */ + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *getattribute = read_obj(&this_instr[6].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert((oparg & 1) == 0); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + PyTypeObject *cls = Py_TYPE(owner_o); + assert(type_version != 0); + DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)getattribute; + assert(func_version != 0); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 2); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( + tstate, PyStackRef_FromPyObjectNew(f), 2, frame); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). + STACK_SHRINK(1); + new_frame->localsplus[0] = owner; + new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); + frame->return_offset = 10 ; + DISPATCH_INLINED(new_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - // _CHECK_MANAGED_OBJECT_HAS_VALUES - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); - } - // _LOAD_ATTR_INSTANCE_VALUE - { - uint16_t offset = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = *value_ptr; - DEOPT_IF(attr_o == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); - null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectSteal(attr_o); - PyStackRef_CLOSE(owner); - } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _CHECK_MANAGED_OBJECT_HAS_VALUES + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + } + // _LOAD_ATTR_INSTANCE_VALUE + { + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = *value_ptr; + DEOPT_IF(attr_o == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr_o); + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectSteal(attr_o); + PyStackRef_CLOSE(owner); } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - // _CHECK_ATTR_METHOD_LAZY_DICT - { - uint16_t dictoffset = read_u16(&this_instr[4].cache); - char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = *(PyObject **)ptr; - /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR); - } - /* Skip 1 cache entry */ - // _LOAD_ATTR_METHOD_LAZY_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _CHECK_ATTR_METHOD_LAZY_DICT + { + uint16_t dictoffset = read_u16(&this_instr[4].cache); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = *(PyObject **)ptr; + /* This object has a __dict__, just not yet created */ + DEOPT_IF(dict != NULL, LOAD_ATTR); } + /* Skip 1 cache entry */ + // _LOAD_ATTR_METHOD_LAZY_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_METHOD_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); - } - // _LOAD_ATTR_METHOD_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - /* Cached method object */ - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + } + // _LOAD_ATTR_METHOD_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + /* Cached method object */ + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_MODULE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - PyDictKeysObject *mod_keys; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _CHECK_ATTR_MODULE_PUSH_KEYS - { - owner = stack_pointer[-1]; - uint32_t dict_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; - assert(dict != NULL); - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR); - mod_keys = keys; - } - // _LOAD_ATTR_MODULE_FROM_KEYS - { - uint16_t index = read_u16(&this_instr[4].cache); - assert(mod_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries)); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); - // Clear mod_keys from stack in case we need to deopt - DEOPT_IF(attr_o == NULL, LOAD_ATTR); - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); - if (!increfed) { - DEOPT_IF(true, LOAD_ATTR); - } - #else - Py_INCREF(attr_o); - attr = PyStackRef_FromPyObjectSteal(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; - PyStackRef_CLOSE(owner); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + PyDictKeysObject *mod_keys; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_MODULE_PUSH_KEYS + { + owner = stack_pointer[-1]; + uint32_t dict_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; + assert(dict != NULL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR); + mod_keys = keys; + } + // _LOAD_ATTR_MODULE_FROM_KEYS + { + uint16_t index = read_u16(&this_instr[4].cache); + assert(mod_keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries)); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); + // Clear mod_keys from stack in case we need to deopt + DEOPT_IF(attr_o == NULL, LOAD_ATTR); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + DEOPT_IF(true, LOAD_ATTR); } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + #else + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - PyStackRef_CLOSE(owner); - attr = PyStackRef_FromPyObjectNew(descr); - } - stack_pointer[-1] = attr; - DISPATCH(); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + PyStackRef_CLOSE(owner); + attr = PyStackRef_FromPyObjectNew(descr); } + stack_pointer[-1] = attr; + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); - } - // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - PyStackRef_CLOSE(owner); - attr = PyStackRef_FromPyObjectNew(descr); - } - stack_pointer[-1] = attr; - DISPATCH(); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + } + // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + PyStackRef_CLOSE(owner); + attr = PyStackRef_FromPyObjectNew(descr); } + stack_pointer[-1] = attr; + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - } - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_PROPERTY_FRAME - { - PyObject *fget = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - assert(Py_IS_TYPE(fget, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)fget; - PyCodeObject *code = (PyCodeObject *)f->func_code; - DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR); - DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR); - DEOPT_IF(code->co_argcount != 1, LOAD_ATTR); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); - new_frame->localsplus[0] = owner; - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - DISPATCH(); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + } + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_PROPERTY_FRAME + { + PyObject *fget = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_IS_TYPE(fget, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)fget; + PyCodeObject *code = (PyCodeObject *)f->func_code; + DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR); + DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR); + DEOPT_IF(code->co_argcount != 1, LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); + new_frame->localsplus[0] = owner; + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_SLOT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - // _LOAD_ATTR_SLOT - { - uint16_t index = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - char *addr = (char *)owner_o + index; - PyObject *attr_o = *(PyObject **)addr; - DEOPT_IF(attr_o == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectNew(attr_o); - PyStackRef_CLOSE(owner); - } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _LOAD_ATTR_SLOT + { + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + char *addr = (char *)owner_o + index; + PyObject *attr_o = *(PyObject **)addr; + DEOPT_IF(attr_o == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectNew(attr_o); + PyStackRef_CLOSE(owner); } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - // _CHECK_ATTR_WITH_HINT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, LOAD_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); - } - // _LOAD_ATTR_WITH_HINT - { - uint16_t hint = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject *attr_o; - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), LOAD_ATTR); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); - attr_o = ep->me_value; - DEOPT_IF(attr_o == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); - attr = PyStackRef_FromPyObjectSteal(attr_o); - null = PyStackRef_NULL; - PyStackRef_CLOSE(owner); - } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _CHECK_ATTR_WITH_HINT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + DEOPT_IF(dict == NULL, LOAD_ATTR); + assert(PyDict_CheckExact((PyObject *)dict)); + } + // _LOAD_ATTR_WITH_HINT + { + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject *attr_o; + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), LOAD_ATTR); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, LOAD_ATTR); + attr_o = ep->me_value; + DEOPT_IF(attr_o == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_BUILD_CLASS); - _PyStackRef bc; - PyObject *bc_o; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_BUILD_CLASS); + _PyStackRef bc; + PyObject *bc_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) CEVAL_GOTO(error); + if (bc_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) CEVAL_GOTO(error); - if (bc_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_NameError, + _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); - } - bc = PyStackRef_FromPyObjectSteal(bc_o); - stack_pointer[0] = bc; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); } + bc = PyStackRef_FromPyObjectSteal(bc_o); + stack_pointer[0] = bc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); - _PyStackRef value; - // Keep in sync with _common_constants in opcode.py - // If we ever have more than two constants, use a lookup table - PyObject *val; - if (oparg == CONSTANT_ASSERTIONERROR) { - val = PyExc_AssertionError; - } - else { - assert(oparg == CONSTANT_NOTIMPLEMENTEDERROR); - val = PyExc_NotImplementedError; - } - value = PyStackRef_FromPyObjectImmortal(val); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); + _PyStackRef value; + // Keep in sync with _common_constants in opcode.py + // If we ever have more than two constants, use a lookup table + PyObject *val; + if (oparg == CONSTANT_ASSERTIONERROR) { + val = PyExc_AssertionError; + } + else { + assert(oparg == CONSTANT_NOTIMPLEMENTEDERROR); + val = PyExc_NotImplementedError; + } + value = PyStackRef_FromPyObjectImmortal(val); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST); - PREDICTED(LOAD_CONST); - _PyStackRef value; - value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST); + PREDICTED(LOAD_CONST); + _PyStackRef value; + value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST_IMMORTAL); - static_assert(0 == 0, "incorrect cache size"); - _PyStackRef value; - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - assert(_Py_IsImmortal(obj)); - value = PyStackRef_FromPyObjectImmortal(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST_IMMORTAL); + static_assert(0 == 0, "incorrect cache size"); + _PyStackRef value; + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + assert(_Py_IsImmortal(obj)); + value = PyStackRef_FromPyObjectImmortal(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_DEREF); - _PyStackRef value; - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *value_o = PyCell_GetRef(cell); - if (value_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); - } - value = PyStackRef_FromPyObjectSteal(value_o); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_DEREF); + _PyStackRef value; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *value_o = PyCell_GetRef(cell); + if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); } + value = PyStackRef_FromPyObjectSteal(value_o); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST); - _PyStackRef value; - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST); + _PyStackRef value; + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); - _PyStackRef value; - value = GETLOCAL(oparg); - // do not use SETLOCAL here, it decrefs the old value - GETLOCAL(oparg) = PyStackRef_NULL; - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); + _PyStackRef value; + value = GETLOCAL(oparg); + // do not use SETLOCAL here, it decrefs the old value + GETLOCAL(oparg) = PyStackRef_NULL; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_CHECK); - _PyStackRef value; - _PyStackRef value_s = GETLOCAL(oparg); - if (PyStackRef_IsNull(value_s)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); - } - value = PyStackRef_DUP(value_s); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_CHECK); + _PyStackRef value; + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); } + value = PyStackRef_DUP(value_s); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); - _PyStackRef value1; - _PyStackRef value2; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - value1 = PyStackRef_DUP(GETLOCAL(oparg1)); - value2 = PyStackRef_DUP(GETLOCAL(oparg2)); - stack_pointer[0] = value1; - stack_pointer[1] = value2; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); + _PyStackRef value1; + _PyStackRef value2; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = PyStackRef_DUP(GETLOCAL(oparg1)); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); - _PyStackRef class_dict_st; - _PyStackRef value; - class_dict_st = stack_pointer[-1]; - PyObject *value_o; - PyObject *name; - PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); - assert(class_dict); - assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); - name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); + _PyStackRef class_dict_st; + _PyStackRef value; + class_dict_st = stack_pointer[-1]; + PyObject *value_o; + PyObject *name; + PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); + assert(class_dict); + assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + CEVAL_GOTO(error); + } + if (!value_o) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + value_o = PyCell_GetRef(cell); + if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); CEVAL_GOTO(error); } - if (!value_o) { - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - value_o = PyCell_GetRef(cell); - if (value_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); - } - } - PyStackRef_CLOSE(class_dict_st); - value = PyStackRef_FromPyObjectSteal(value_o); - stack_pointer[-1] = value; - DISPATCH(); } + PyStackRef_CLOSE(class_dict_st); + value = PyStackRef_FromPyObjectSteal(value_o); + stack_pointer[-1] = value; + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); - _PyStackRef mod_or_class_dict; - _PyStackRef v; - mod_or_class_dict = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *v_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(mod_or_class_dict); - if (err < 0) CEVAL_GOTO(pop_1_error); - if (v_o == NULL) { - if (PyDict_CheckExact(GLOBALS()) + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); + _PyStackRef mod_or_class_dict; + _PyStackRef v; + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *v_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(mod_or_class_dict); + if (err < 0) CEVAL_GOTO(pop_1_error); + if (v_o == NULL) { + if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) - { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - CEVAL_GOTO(error); + { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); } + CEVAL_GOTO(error); } - else { - /* Slow-path if globals or builtins is not a dict */ - /* namespace 1: globals */ - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + } + else { + /* Slow-path if globals or builtins is not a dict */ + /* namespace 1: globals */ + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) CEVAL_GOTO(error); + if (v_o == NULL) { + /* namespace 2: builtins */ _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); + int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) CEVAL_GOTO(error); if (v_o == NULL) { - /* namespace 2: builtins */ _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) CEVAL_GOTO(error); - if (v_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); - } + CEVAL_GOTO(error); } } - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); } - v = PyStackRef_FromPyObjectSteal(v_o); - stack_pointer[-1] = v; - DISPATCH(); + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[-1] = v; + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL); - PREDICTED(LOAD_GLOBAL); - _Py_CODEUNIT* const this_instr = next_instr - 5; - (void)this_instr; - _PyStackRef *res; - _PyStackRef null = PyStackRef_NULL; - // _SPECIALIZE_LOAD_GLOBAL - { - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_GLOBAL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 1 cache entry */ - /* Skip 1 cache entry */ - /* Skip 1 cache entry */ - // _LOAD_GLOBAL - { - res = &stack_pointer[0]; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL); + PREDICTED(LOAD_GLOBAL); + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + _PyStackRef *res; + _PyStackRef null = PyStackRef_NULL; + // _SPECIALIZE_LOAD_GLOBAL + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); + _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*res)) CEVAL_GOTO(error); - null = PyStackRef_NULL; + DISPATCH_SAME_OPARG(); } - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + OPCODE_DEFERRED_INC(LOAD_GLOBAL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + // _LOAD_GLOBAL + { + res = &stack_pointer[0]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(*res)) CEVAL_GOTO(error); + null = PyStackRef_NULL; + } + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - PyDictKeysObject *builtins_keys; - _PyStackRef res; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_GLOBALS_VERSION - { - uint16_t version = read_u16(&this_instr[2].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(keys)); - } - // _GUARD_BUILTINS_VERSION_PUSH_KEYS - { - uint16_t version = read_u16(&this_instr[3].cache); - PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); - builtins_keys = keys; - assert(DK_IS_UNICODE(builtins_keys)); - } - // _LOAD_GLOBAL_BUILTINS_FROM_KEYS - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL); - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL); - #else - Py_INCREF(res_o); - res = PyStackRef_FromPyObjectSteal(res_o); - #endif - STAT_INC(LOAD_GLOBAL, hit); - null = PyStackRef_NULL; - } - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + PyDictKeysObject *builtins_keys; + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION + { + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(keys)); + } + // _GUARD_BUILTINS_VERSION_PUSH_KEYS + { + uint16_t version = read_u16(&this_instr[3].cache); + PyDictObject *dict = (PyDictObject *)BUILTINS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + builtins_keys = keys; + assert(DK_IS_UNICODE(builtins_keys)); + } + // _LOAD_GLOBAL_BUILTINS_FROM_KEYS + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + DEOPT_IF(!increfed, LOAD_GLOBAL); + #else + Py_INCREF(res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + null = PyStackRef_NULL; } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - PyDictKeysObject *globals_keys; - _PyStackRef res; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_GLOBALS_VERSION_PUSH_KEYS - { - uint16_t version = read_u16(&this_instr[2].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); - globals_keys = keys; - assert(DK_IS_UNICODE(globals_keys)); - } - /* Skip 1 cache entry */ - // _LOAD_GLOBAL_MODULE_FROM_KEYS - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL); - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL); - #else - Py_INCREF(res_o); - res = PyStackRef_FromPyObjectSteal(res_o); - #endif - STAT_INC(LOAD_GLOBAL, hit); - null = PyStackRef_NULL; - } - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + PyDictKeysObject *globals_keys; + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION_PUSH_KEYS + { + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + globals_keys = keys; + assert(DK_IS_UNICODE(globals_keys)); + } + /* Skip 1 cache entry */ + // _LOAD_GLOBAL_MODULE_FROM_KEYS + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + DEOPT_IF(!increfed, LOAD_GLOBAL); + #else + Py_INCREF(res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + null = PyStackRef_NULL; } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_LOCALS); - _PyStackRef locals; - PyObject *l = LOCALS(); - if (l == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_SystemError, + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_LOCALS); + _PyStackRef locals; + PyObject *l = LOCALS(); + if (l == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); - } - locals = PyStackRef_FromPyObjectNew(l); - stack_pointer[0] = locals; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); } + locals = PyStackRef_FromPyObjectNew(l); + stack_pointer[0] = locals; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_NAME); - _PyStackRef v; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *v_o = _PyEval_LoadName(tstate, frame, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) CEVAL_GOTO(error); - v = PyStackRef_FromPyObjectSteal(v_o); - stack_pointer[0] = v; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_NAME); + _PyStackRef v; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) CEVAL_GOTO(error); + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_SMALL_INT); - _PyStackRef value; - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectImmortal(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SMALL_INT); + _PyStackRef value; + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectImmortal(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_SPECIAL); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self_or_null; - owner = stack_pointer[-1]; - assert(oparg <= SPECIAL_MAX); - PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); - PyObject *name = _Py_SpecialMethods[oparg].name; - PyObject *self_or_null_o; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = _PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SPECIAL); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self_or_null; + owner = stack_pointer[-1]; + assert(oparg <= SPECIAL_MAX); + PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); + PyObject *name = _Py_SpecialMethods[oparg].name; + PyObject *self_or_null_o; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = _PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, _Py_SpecialMethods[oparg].error, Py_TYPE(owner_o)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - CEVAL_GOTO(error); + stack_pointer = _PyFrame_GetStackPointer(frame); } - attr = PyStackRef_FromPyObjectSteal(attr_o); - self_or_null = self_or_null_o == NULL ? - PyStackRef_NULL : PyStackRef_FromPyObjectSteal(self_or_null_o); - stack_pointer[0] = attr; - stack_pointer[1] = self_or_null; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + CEVAL_GOTO(error); } + attr = PyStackRef_FromPyObjectSteal(attr_o); + self_or_null = self_or_null_o == NULL ? + PyStackRef_NULL : PyStackRef_FromPyObjectSteal(self_or_null_o); + stack_pointer[0] = attr; + stack_pointer[1] = self_or_null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR); - PREDICTED(LOAD_SUPER_ATTR); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - // _SPECIALIZE_LOAD_SUPER_ATTR - { - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - int load_method = oparg & 1; - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _LOAD_SUPER_ATTR - { - self_st = stack_pointer[-1]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - CEVAL_GOTO(pop_3_error); - } - } - // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = {class, self}; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(super); - } - } - } - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - if (super == NULL) CEVAL_GOTO(pop_3_error); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR); + PREDICTED(LOAD_SUPER_ATTR); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + // _SPECIALIZE_LOAD_SUPER_ATTR + { + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + int load_method = oparg & 1; + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(super, name); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); stack_pointer = _PyFrame_GetStackPointer(frame); - Py_DECREF(super); - if (attr_o == NULL) CEVAL_GOTO(error); - attr = PyStackRef_FromPyObjectSteal(attr_o); - null = PyStackRef_NULL; - } - stack_pointer[0] = attr; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr_st; - /* Skip 1 cache entry */ + // _LOAD_SUPER_ATTR + { self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + CEVAL_GOTO(pop_3_error); + } + } + // we make no attempt to optimize here; specializations should + // handle any case whose performance we care about + PyObject *stack[] = {class, self}; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(super); + } + } + } PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); - if (attr == NULL) CEVAL_GOTO(pop_3_error); - attr_st = PyStackRef_FromPyObjectSteal(attr); - stack_pointer[-3] = attr_st; - stack_pointer += -2; + if (super == NULL) CEVAL_GOTO(pop_3_error); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(super); + if (attr_o == NULL) CEVAL_GOTO(error); + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; } + stack_pointer[0] = attr; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr_st; + /* Skip 1 cache entry */ + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(!(oparg & 1)); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + if (attr == NULL) CEVAL_GOTO(pop_3_error); + attr_st = PyStackRef_FromPyObjectSteal(attr); + stack_pointer[-3] = attr_st; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef self_or_null; - /* Skip 1 cache entry */ - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - PyTypeObject *cls = (PyTypeObject *)class; - int method_found = 0; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - if (attr_o == NULL) { - PyStackRef_CLOSE(self_st); - CEVAL_GOTO(pop_3_error); - } - if (method_found) { - self_or_null = self_st; // transfer ownership - } else { - PyStackRef_CLOSE(self_st); - self_or_null = PyStackRef_NULL; - } - attr = PyStackRef_FromPyObjectSteal(attr_o); - stack_pointer[-3] = attr; - stack_pointer[-2] = self_or_null; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef self_or_null; + /* Skip 1 cache entry */ + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(oparg & 1); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyTypeObject *cls = (PyTypeObject *)class; + int method_found = 0; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + if (attr_o == NULL) { + PyStackRef_CLOSE(self_st); + CEVAL_GOTO(pop_3_error); + } + if (method_found) { + self_or_null = self_st; // transfer ownership + } else { + PyStackRef_CLOSE(self_st); + self_or_null = PyStackRef_NULL; } + attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_CELL); - // "initial" is probably NULL but not if it's an arg (or set - // via the f_locals proxy before MAKE_CELL has run). - PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *cell = PyCell_New(initial); - if (cell == NULL) { - CEVAL_GOTO(error); - } - SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell)); - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_CELL); + // "initial" is probably NULL but not if it's an arg (or set + // via the f_locals proxy before MAKE_CELL has run). + PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *cell = PyCell_New(initial); + if (cell == NULL) { + CEVAL_GOTO(error); } + SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell)); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_FUNCTION); - _PyStackRef codeobj_st; - _PyStackRef func; - codeobj_st = stack_pointer[-1]; - PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(codeobj_st); - if (func_obj == NULL) CEVAL_GOTO(pop_1_error); - _PyFunction_SetVersion( + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_FUNCTION); + _PyStackRef codeobj_st; + _PyStackRef func; + codeobj_st = stack_pointer[-1]; + PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(codeobj_st); + if (func_obj == NULL) CEVAL_GOTO(pop_1_error); + _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); - func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); - stack_pointer[-1] = func; - DISPATCH(); - } + func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); + stack_pointer[-1] = func; + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAP_ADD); - _PyStackRef dict_st; - _PyStackRef key; - _PyStackRef value; - value = stack_pointer[-1]; - key = stack_pointer[-2]; - dict_st = stack_pointer[-3 - (oparg - 1)]; - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); - /* dict[key] = value */ - // Do not DECREF INPUTS because the function steals the references - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2( - (PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(key), - PyStackRef_AsPyObjectSteal(value) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(pop_2_error); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAP_ADD); + _PyStackRef dict_st; + _PyStackRef key; + _PyStackRef value; + value = stack_pointer[-1]; + key = stack_pointer[-2]; + dict_st = stack_pointer[-3 - (oparg - 1)]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + /* dict[key] = value */ + // Do not DECREF INPUTS because the function steals the references + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2( + (PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(key), + PyStackRef_AsPyObjectSteal(value) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) CEVAL_GOTO(pop_2_error); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_CLASS); - _PyStackRef subject; - _PyStackRef type; - _PyStackRef names; - _PyStackRef attrs; - names = stack_pointer[-1]; - type = stack_pointer[-2]; - subject = stack_pointer[-3]; - // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or - // None on failure. - assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attrs_o = _PyEval_MatchClass(tstate, - PyStackRef_AsPyObjectBorrow(subject), - PyStackRef_AsPyObjectBorrow(type), oparg, - PyStackRef_AsPyObjectBorrow(names)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(subject); - PyStackRef_CLOSE(type); - PyStackRef_CLOSE(names); - if (attrs_o) { - assert(PyTuple_CheckExact(attrs_o)); // Success! - attrs = PyStackRef_FromPyObjectSteal(attrs_o); - } - else { - if (_PyErr_Occurred(tstate)) CEVAL_GOTO(pop_3_error); - // Error! - attrs = PyStackRef_None; // Failure! - } - stack_pointer[-3] = attrs; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_CLASS); + _PyStackRef subject; + _PyStackRef type; + _PyStackRef names; + _PyStackRef attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; + // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or + // None on failure. + assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attrs_o = _PyEval_MatchClass(tstate, + PyStackRef_AsPyObjectBorrow(subject), + PyStackRef_AsPyObjectBorrow(type), oparg, + PyStackRef_AsPyObjectBorrow(names)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(subject); + PyStackRef_CLOSE(type); + PyStackRef_CLOSE(names); + if (attrs_o) { + assert(PyTuple_CheckExact(attrs_o)); // Success! + attrs = PyStackRef_FromPyObjectSteal(attrs_o); + } + else { + if (_PyErr_Occurred(tstate)) CEVAL_GOTO(pop_3_error); + // Error! + attrs = PyStackRef_None; // Failure! + } + stack_pointer[-3] = attrs; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_KEYS); - _PyStackRef subject; - _PyStackRef keys; - _PyStackRef values_or_none; - keys = stack_pointer[-1]; - subject = stack_pointer[-2]; - // On successful match, PUSH(values). Otherwise, PUSH(None). - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, - PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (values_or_none_o == NULL) CEVAL_GOTO(error); - values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); - stack_pointer[0] = values_or_none; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_KEYS); + _PyStackRef subject; + _PyStackRef keys; + _PyStackRef values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; + // On successful match, PUSH(values). Otherwise, PUSH(None). + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, + PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (values_or_none_o == NULL) CEVAL_GOTO(error); + values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); + stack_pointer[0] = values_or_none; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_MAPPING); - _PyStackRef subject; - _PyStackRef res; - subject = stack_pointer[-1]; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_MAPPING); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_SEQUENCE); - _PyStackRef subject; - _PyStackRef res; - subject = stack_pointer[-1]; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_SEQUENCE); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOP); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOP); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOT_TAKEN); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOT_TAKEN); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_EXCEPT); - _PyStackRef exc_value; - exc_value = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_XSETREF(exc_info->exc_value, + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_EXCEPT); + _PyStackRef exc_value; + exc_value = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XSETREF(exc_info->exc_value, PyStackRef_IsNone(exc_value) ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_FALSE); - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_FALSE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NONE); - _PyStackRef value; - _PyStackRef b; - _PyStackRef cond; - /* Skip 1 cache entry */ - // _IS_NONE - { - value = stack_pointer[-1]; - if (PyStackRef_IsNone(value)) { - b = PyStackRef_True; - } - else { - b = PyStackRef_False; - PyStackRef_CLOSE(value); - } + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NONE); + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + { + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; } - // _POP_JUMP_IF_TRUE - { - cond = b; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + else { + b = PyStackRef_False; + PyStackRef_CLOSE(value); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); } + // _POP_JUMP_IF_TRUE + { + cond = b; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); - _PyStackRef value; - _PyStackRef b; - _PyStackRef cond; - /* Skip 1 cache entry */ - // _IS_NONE - { - value = stack_pointer[-1]; - if (PyStackRef_IsNone(value)) { - b = PyStackRef_True; - } - else { - b = PyStackRef_False; - PyStackRef_CLOSE(value); - } + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + { + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; } - // _POP_JUMP_IF_FALSE - { - cond = b; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + else { + b = PyStackRef_False; + PyStackRef_CLOSE(value); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_TRUE); - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; + // _POP_JUMP_IF_FALSE + { + cond = b; assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsTrue(cond); + int flag = PyStackRef_IsFalse(cond); RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS){ + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_TRUE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_TOP); - _PyStackRef value; - value = stack_pointer[-1]; - PyStackRef_CLOSE(value); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_TOP); + _PyStackRef value; + value = stack_pointer[-1]; + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_EXC_INFO); - _PyStackRef exc; - _PyStackRef prev_exc; - _PyStackRef new_exc; - exc = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - if (exc_info->exc_value != NULL) { - prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); - } - else { - prev_exc = PyStackRef_None; - } - assert(PyStackRef_ExceptionInstanceCheck(exc)); - exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); - new_exc = exc; - stack_pointer[-1] = prev_exc; - stack_pointer[0] = new_exc; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_EXC_INFO); + _PyStackRef exc; + _PyStackRef prev_exc; + _PyStackRef new_exc; + exc = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); + } + else { + prev_exc = PyStackRef_None; + } + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_NULL); - _PyStackRef res; - res = PyStackRef_NULL; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_NULL); + _PyStackRef res; + res = PyStackRef_NULL; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(RAISE_VARARGS); - _PyStackRef *args; - args = &stack_pointer[-oparg]; - assert(oparg < 3); - PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; - PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(RAISE_VARARGS); + _PyStackRef *args; + args = &stack_pointer[-oparg]; + assert(oparg < 3); + PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; + PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = do_raise(tstate, exc, cause); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + assert(oparg == 0); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = do_raise(tstate, exc, cause); + monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - assert(oparg == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_reraise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(exception_unwind); - } - CEVAL_GOTO(error); + CEVAL_GOTO(exception_unwind); } + CEVAL_GOTO(error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(RERAISE); - _PyStackRef *values; - _PyStackRef exc_st; - exc_st = stack_pointer[-1]; - values = &stack_pointer[-1 - oparg]; - PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); - assert(oparg >= 0 && oparg <= 2); - if (oparg) { - PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]); - if (PyLong_Check(lasti)) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyLong_AsLong(lasti); - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(!_PyErr_Occurred(tstate)); - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); - stack_pointer = _PyFrame_GetStackPointer(frame); - Py_DECREF(exc); - CEVAL_GOTO(error); - } - stack_pointer += 1; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(RERAISE); + _PyStackRef *values; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); + assert(oparg >= 0 && oparg <= 2); + if (oparg) { + PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]); + if (PyLong_Check(lasti)) { + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyLong_AsLong(lasti); + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!_PyErr_Occurred(tstate)); } - assert(exc && PyExceptionInstance_Check(exc)); - stack_pointer += -1; + else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(exc); + CEVAL_GOTO(error); + } + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(exception_unwind); - DISPATCH(); } + assert(exc && PyExceptionInstance_Check(exc)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(exception_unwind); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESERVED); - assert(0 && "Executing RESERVED instruction."); - Py_FatalError("Executing RESERVED instruction."); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESERVED); + assert(0 && "Executing RESERVED instruction."); + Py_FatalError("Executing RESERVED instruction."); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME); - PREDICTED(RESUME); - _Py_CODEUNIT* this_instr = next_instr - 1; - (void)this_instr; - // _LOAD_BYTECODE - { - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) CEVAL_GOTO(error); - _PyFrame_SetStackPointer(frame, stack_pointer); - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; - frame->instr_ptr = bytecode + off; - // Make sure this_instr gets reset correctley for any uops that - // follow - next_instr = frame->instr_ptr; - DISPATCH(); - } - #endif - } - // _MAYBE_INSTRUMENT - { - if (tstate->tracing == 0) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - CEVAL_GOTO(error); - } - next_instr = this_instr; - - DISPATCH(); - } - } - } - // _QUICKEN_RESUME - { - #if ENABLE_SPECIALIZATION_FT - if (tstate->tracing == 0 && this_instr->op.code == RESUME) { - FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); - } - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _CHECK_PERIODIC_IF_NOT_YIELD_FROM - { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - } - } - } - DISPATCH(); - } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME_CHECK); - static_assert(0 == 0, "incorrect cache size"); - #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; - #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME); + PREDICTED(RESUME); + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; + // _LOAD_BYTECODE + { #ifdef Py_GIL_DISABLED - DEOPT_IF(frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME); - #endif - DISPATCH(); - } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_GENERATOR); - _PyStackRef res; - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (gen == NULL) CEVAL_GOTO(error); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_VALUE); - _PyStackRef retval; - _PyStackRef res; - retval = stack_pointer[-1]; - #if TIER_ONE - assert(frame != entry_frame); - #endif - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(EMPTY()); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(frame->return_offset); - res = temp; - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (bytecode == NULL) CEVAL_GOTO(error); + _PyFrame_SetStackPointer(frame, stack_pointer); + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + // Make sure this_instr gets reset correctley for any uops that + // follow + next_instr = frame->instr_ptr; + DISPATCH(); + } + #endif } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND); - PREDICTED(SEND); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef receiver; - _PyStackRef v; - _PyStackRef retval; - // _SPECIALIZE_SEND - { - receiver = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; + // _MAYBE_INSTRUMENT + { + if (tstate->tracing == 0) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Send(receiver, next_instr); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + if (err) { + CEVAL_GOTO(error); + } + next_instr = this_instr; + DISPATCH(); } - OPCODE_DEFERRED_INC(SEND); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ } - // _SEND - { - v = stack_pointer[-1]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - PyObject *retval_o; - assert(frame != entry_frame); - if ((tstate->interp->eval_frame == NULL) && - (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && - ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) - { - PyGenObject *gen = (PyGenObject *)receiver_o; - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert( 2 + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2 + oparg); - assert(gen_frame->previous == NULL); - gen_frame->previous = frame; - DISPATCH_INLINED(gen_frame); - } - if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = PyObject_CallMethodOneArg(receiver_o, - &_Py_ID(send), - PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - if (retval_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - } + } + // _QUICKEN_RESUME + { + #if ENABLE_SPECIALIZATION_FT + if (tstate->tracing == 0 && this_instr->op.code == RESUME) { + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + } + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyGen_FetchStopIterationValue(&retval_o); + int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err == 0) { - assert(retval_o != NULL); - JUMPBY(oparg); - } - else { - PyStackRef_CLOSE(v); - CEVAL_GOTO(pop_1_error); - } + if (err != 0) CEVAL_GOTO(error); } - PyStackRef_CLOSE(v); - retval = PyStackRef_FromPyObjectSteal(retval_o); } - stack_pointer[-1] = retval; - DISPATCH(); } + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND_GEN); - static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); - _PyStackRef receiver; - _PyStackRef v; - _PyInterpreterFrame *gen_frame; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, SEND); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME_CHECK); + static_assert(0 == 0, "incorrect cache size"); + #if defined(__EMSCRIPTEN__) + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + DEOPT_IF(eval_breaker != version, RESUME); + #ifdef Py_GIL_DISABLED + DEOPT_IF(frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME); + #endif + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_GENERATOR); + _PyStackRef res; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (gen == NULL) CEVAL_GOTO(error); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectSteal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_VALUE); + _PyStackRef retval; + _PyStackRef res; + retval = stack_pointer[-1]; + #if TIER_ONE + assert(frame != entry_frame); + #endif + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(frame->return_offset); + res = temp; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND); + PREDICTED(SEND); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef receiver; + _PyStackRef v; + _PyStackRef retval; + // _SPECIALIZE_SEND + { + receiver = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Send(receiver, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); } - // _SEND_GEN_FRAME + OPCODE_DEFERRED_INC(SEND); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _SEND + { + v = stack_pointer[-1]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + PyObject *retval_o; + assert(frame != entry_frame); + if ((tstate->interp->eval_frame == NULL) && + (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && + ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) { - v = stack_pointer[-1]; - receiver = stack_pointer[-2]; - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); - STAT_INC(SEND, hit); - gen_frame = &gen->gi_iframe; + PyGenObject *gen = (PyGenObject *)receiver_o; + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + STACK_SHRINK(1); _PyFrame_StackPush(gen_frame, v); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; assert( 2 + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)( 2 + oparg); + assert(gen_frame->previous == NULL); gen_frame->previous = frame; + DISPATCH_INLINED(gen_frame); } - // _PUSH_FRAME - { - new_frame = gen_frame; - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); + retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); + stack_pointer = _PyFrame_GetStackPointer(frame); } - DISPATCH(); - } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SETUP_ANNOTATIONS); - PyObject *ann_dict; - if (LOCALS() == NULL) { + else { _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when setting up annotations"); + retval_o = PyObject_CallMethodOneArg(receiver_o, + &_Py_ID(send), + PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); } - /* check if __annotations__ in locals()... */ - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) CEVAL_GOTO(error); - if (ann_dict == NULL) { + if (retval_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - ann_dict = PyDict_New(); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); stack_pointer = _PyFrame_GetStackPointer(frame); - if (ann_dict == NULL) CEVAL_GOTO(error); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + } _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); + int err = _PyGen_FetchStopIterationValue(&retval_o); stack_pointer = _PyFrame_GetStackPointer(frame); - Py_DECREF(ann_dict); - if (err) CEVAL_GOTO(error); - } - else { - Py_DECREF(ann_dict); + if (err == 0) { + assert(retval_o != NULL); + JUMPBY(oparg); + } + else { + PyStackRef_CLOSE(v); + CEVAL_GOTO(pop_1_error); + } } - DISPATCH(); + PyStackRef_CLOSE(v); + retval = PyStackRef_FromPyObjectSteal(retval_o); } + stack_pointer[-1] = retval; + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_ADD); - _PyStackRef set; - _PyStackRef v; +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_GEN); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef receiver; + _PyStackRef v; + _PyInterpreterFrame *gen_frame; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, SEND); + } + // _SEND_GEN_FRAME + { v = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), - PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - if (err) CEVAL_GOTO(pop_1_error); + receiver = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + STAT_INC(SEND, hit); + gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2 + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2 + oparg); + gen_frame->previous = frame; + } + // _PUSH_FRAME + { + new_frame = gen_frame; + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); } + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); - _PyStackRef attr_st; - _PyStackRef func_in; - _PyStackRef func_out; - func_in = stack_pointer[-1]; - attr_st = stack_pointer[-2]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); - PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); - func_out = func_in; - assert(PyFunction_Check(func)); - size_t offset = _Py_FunctionAttributeOffsets[oparg]; - assert(offset != 0); - PyObject **ptr = (PyObject **)(((char *)func) + offset); - assert(*ptr == NULL); - *ptr = attr; - stack_pointer[-2] = func_out; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SETUP_ANNOTATIONS); + PyObject *ann_dict; + if (LOCALS() == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when setting up annotations"); + stack_pointer = _PyFrame_GetStackPointer(frame); + CEVAL_GOTO(error); + } + /* check if __annotations__ in locals()... */ + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) CEVAL_GOTO(error); + if (ann_dict == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + ann_dict = PyDict_New(); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (ann_dict == NULL) CEVAL_GOTO(error); + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(ann_dict); + if (err) CEVAL_GOTO(error); } + else { + Py_DECREF(ann_dict); + } + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_ADD); + _PyStackRef set; + _PyStackRef v; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + if (err) CEVAL_GOTO(pop_1_error); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); + _PyStackRef attr_st; + _PyStackRef func_in; + _PyStackRef func_out; + func_in = stack_pointer[-1]; + attr_st = stack_pointer[-2]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); + PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); + func_out = func_in; + assert(PyFunction_Check(func)); + size_t offset = _Py_FunctionAttributeOffsets[oparg]; + assert(offset != 0); + PyObject **ptr = (PyObject **)(((char *)func) + offset); + assert(*ptr == NULL); + *ptr = attr; + stack_pointer[-2] = func_out; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_UPDATE); - _PyStackRef set; - _PyStackRef iterable; - iterable = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_UPDATE); + _PyStackRef set; + _PyStackRef iterable; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(iterable); - if (err < 0) CEVAL_GOTO(pop_1_error); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(iterable); + if (err < 0) CEVAL_GOTO(pop_1_error); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR); - PREDICTED(STORE_ATTR); - _Py_CODEUNIT* const this_instr = next_instr - 5; - (void)this_instr; - _PyStackRef owner; - _PyStackRef v; - // _SPECIALIZE_STORE_ATTR - { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_StoreAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(STORE_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 3 cache entries */ - // _STORE_ATTR - { - v = stack_pointer[-2]; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR); + PREDICTED(STORE_ATTR); + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + _PyStackRef owner; + _PyStackRef v; + // _SPECIALIZE_STORE_ATTR + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), - name, PyStackRef_AsPyObjectBorrow(v)); + _Py_Specialize_StoreAttr(owner, next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(owner); - if (err) CEVAL_GOTO(pop_2_error); + DISPATCH_SAME_OPARG(); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + OPCODE_DEFERRED_INC(STORE_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 3 cache entries */ + // _STORE_ATTR + { + v = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(owner); + if (err) CEVAL_GOTO(pop_2_error); } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION_AND_LOCK - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(type_version != 0); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); - PyTypeObject *tp = Py_TYPE(owner_o); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR); - } + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION_AND_LOCK + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + DEOPT_IF(true, STORE_ATTR); } - // _GUARD_DORV_NO_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (_PyObject_GetManagedDict(owner_o) || + } + // _GUARD_DORV_NO_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { - UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR); - } - } - // _STORE_ATTR_INSTANCE_VALUE - { - value = stack_pointer[-2]; - uint16_t offset = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - STAT_INC(STORE_ATTR, hit); - assert(_PyObject_GetManagedDict(owner_o) == NULL); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *old_value = *value_ptr; - FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); - if (old_value == NULL) { - PyDictValues *values = _PyObject_InlineValues(owner_o); - Py_ssize_t index = value_ptr - values->values; - _PyDictValues_AddToInsertionOrder(values, index); - } UNLOCK_OBJECT(owner_o); - Py_XDECREF(old_value); - PyStackRef_CLOSE(owner); + DEOPT_IF(true, STORE_ATTR); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); } + // _STORE_ATTR_INSTANCE_VALUE + { + value = stack_pointer[-2]; + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + STAT_INC(STORE_ATTR, hit); + assert(_PyObject_GetManagedDict(owner_o) == NULL); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *old_value = *value_ptr; + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); + if (old_value == NULL) { + PyDictValues *values = _PyObject_InlineValues(owner_o); + Py_ssize_t index = value_ptr - values->values; + _PyDictValues_AddToInsertionOrder(values, index); + } + UNLOCK_OBJECT(owner_o); + Py_XDECREF(old_value); + PyStackRef_CLOSE(owner); + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_SLOT); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); - } - // _STORE_ATTR_SLOT - { - value = stack_pointer[-2]; - uint16_t index = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); - char *addr = (char *)owner_o + index; - STAT_INC(STORE_ATTR, hit); - PyObject *old_value = *(PyObject **)addr; - FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); - UNLOCK_OBJECT(owner_o); - Py_XDECREF(old_value); - PyStackRef_CLOSE(owner); - } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + } + // _STORE_ATTR_SLOT + { + value = stack_pointer[-2]; + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + char *addr = (char *)owner_o + index; + STAT_INC(STORE_ATTR, hit); + PyObject *old_value = *(PyObject **)addr; + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); + Py_XDECREF(old_value); + PyStackRef_CLOSE(owner); } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + } + // _STORE_ATTR_WITH_HINT + { + value = stack_pointer[-2]; + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + DEOPT_IF(dict == NULL, STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR); + #ifdef Py_GIL_DISABLED + if (dict != _PyObject_GetManagedDict(owner_o)) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); } - // _STORE_ATTR_WITH_HINT - { - value = stack_pointer[-2]; - uint16_t hint = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, STORE_ATTR); - DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR); - #ifdef Py_GIL_DISABLED - if (dict != _PyObject_GetManagedDict(owner_o)) { - UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); - } - #endif - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (hint >= (size_t)dict->ma_keys->dk_nentries || + #endif + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (hint >= (size_t)dict->ma_keys->dk_nentries || !DK_IS_UNICODE(dict->ma_keys)) { - UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); - } - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) { - UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); - } - PyObject *old_value = ep->me_value; - if (old_value == NULL) { - UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(dict); - // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, - // when dict only holds the strong reference to value in ep->me_value. - Py_XDECREF(old_value); - STAT_INC(STORE_ATTR, hit); - PyStackRef_CLOSE(owner); + DEOPT_IF(true, STORE_ATTR); + } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } + PyObject *old_value = ep->me_value; + if (old_value == NULL) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_DEREF); - _PyStackRef v; - v = stack_pointer[-1]; - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); _PyFrame_SetStackPointer(frame, stack_pointer); - PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); + _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(dict); + // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, + // when dict only holds the strong reference to value in ep->me_value. + Py_XDECREF(old_value); + STAT_INC(STORE_ATTR, hit); + PyStackRef_CLOSE(owner); } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST); - _PyStackRef value; - value = stack_pointer[-1]; - SETLOCAL(oparg, value); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_DEREF); + _PyStackRef v; + v = stack_pointer[-1]; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST); + _PyStackRef value; + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); - _PyStackRef value1; - _PyStackRef value2; - value1 = stack_pointer[-1]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - SETLOCAL(oparg1, value1); - value2 = PyStackRef_DUP(GETLOCAL(oparg2)); - stack_pointer[-1] = value2; - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); + _PyStackRef value1; + _PyStackRef value2; + value1 = stack_pointer[-1]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[-1] = value2; + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_STORE_FAST); - _PyStackRef value2; - _PyStackRef value1; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - SETLOCAL(oparg1, value1); - SETLOCAL(oparg2, value2); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_STORE_FAST); + _PyStackRef value2; + _PyStackRef value1; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + SETLOCAL(oparg2, value2); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_GLOBAL); - _PyStackRef v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - if (err) CEVAL_GOTO(pop_1_error); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_GLOBAL); + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + if (err) CEVAL_GOTO(pop_1_error); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_NAME); - _PyStackRef v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_NAME); + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - CEVAL_GOTO(pop_1_error); - } - if (PyDict_CheckExact(ns)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); - if (err) CEVAL_GOTO(pop_1_error); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + CEVAL_GOTO(pop_1_error); + } + if (PyDict_CheckExact(ns)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); } + PyStackRef_CLOSE(v); + if (err) CEVAL_GOTO(pop_1_error); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_SLICE); - _PyStackRef v; - _PyStackRef container; - _PyStackRef start; - _PyStackRef stop; - // _SPECIALIZE_STORE_SLICE - { - // Placeholder until we implement STORE_SLICE specialization - #if ENABLE_SPECIALIZATION - OPCODE_DEFERRED_INC(STORE_SLICE); - #endif /* ENABLE_SPECIALIZATION */ + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_SLICE); + _PyStackRef v; + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + // _SPECIALIZE_STORE_SLICE + { + // Placeholder until we implement STORE_SLICE specialization + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(STORE_SLICE); + #endif /* ENABLE_SPECIALIZATION */ + } + // _STORE_SLICE + { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); + int err; + if (slice == NULL) { + err = 1; } - // _STORE_SLICE - { - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - v = stack_pointer[-4]; + else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); - int err; - if (slice == NULL) { - err = 1; - } - else { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - Py_DECREF(slice); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - } - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(container); - if (err) CEVAL_GOTO(pop_4_error); + Py_DECREF(slice); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); + if (err) CEVAL_GOTO(pop_4_error); } + stack_pointer += -4; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR); - PREDICTED(STORE_SUBSCR); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef container; - _PyStackRef sub; - _PyStackRef v; - // _SPECIALIZE_STORE_SUBSCR - { - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_StoreSubscr(container, sub, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(STORE_SUBSCR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _STORE_SUBSCR - { - v = stack_pointer[-3]; - /* container[sub] = v */ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR); + PREDICTED(STORE_SUBSCR); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef container; + _PyStackRef sub; + _PyStackRef v; + // _SPECIALIZE_STORE_SUBSCR + { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + _Py_Specialize_StoreSubscr(container, sub, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (err) CEVAL_GOTO(pop_3_error); + DISPATCH_SAME_OPARG(); } - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + OPCODE_DEFERRED_INC(STORE_SUBSCR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_DICT); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef dict_st; - _PyStackRef sub; - /* Skip 1 cache entry */ - sub = stack_pointer[-1]; - dict_st = stack_pointer[-2]; - value = stack_pointer[-3]; - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); - STAT_INC(STORE_SUBSCR, hit); + // _STORE_SUBSCR + { + v = stack_pointer[-3]; + /* container[sub] = v */ _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(sub), - PyStackRef_AsPyObjectSteal(value)); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(dict_st); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); if (err) CEVAL_GOTO(pop_3_error); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); } + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef dict_st; + _PyStackRef sub; + /* Skip 1 cache entry */ + sub = stack_pointer[-1]; + dict_st = stack_pointer[-2]; + value = stack_pointer[-3]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + STAT_INC(STORE_SUBSCR, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(sub), + PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(dict_st); + if (err) CEVAL_GOTO(pop_3_error); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef list_st; - _PyStackRef sub_st; - /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - value = stack_pointer[-3]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); - // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR); - // Ensure index < len(list) - if (index >= PyList_GET_SIZE(list)) { - UNLOCK_OBJECT(list); - DEOPT_IF(true, STORE_SUBSCR); - } - STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); - PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); - assert(old_value != NULL); - UNLOCK_OBJECT(list); // unlock before decrefs! - Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - PyStackRef_CLOSE(list_st); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef list_st; + _PyStackRef sub_st; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + value = stack_pointer[-3]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + // Ensure nonnegative, zero-or-one-digit ints. + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR); + // Ensure index < len(list) + if (index >= PyList_GET_SIZE(list)) { + UNLOCK_OBJECT(list); + DEOPT_IF(true, STORE_SUBSCR); + } + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); + assert(old_value != NULL); + UNLOCK_OBJECT(list); // unlock before decrefs! + Py_DECREF(old_value); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(list_st); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SWAP); - _PyStackRef bottom_in; - _PyStackRef top_in; - _PyStackRef top_out; - _PyStackRef bottom_out; - top_in = stack_pointer[-1]; - bottom_in = stack_pointer[-2 - (oparg-2)]; - bottom_out = bottom_in; - top_out = top_in; - assert(oparg >= 2); - stack_pointer[-2 - (oparg-2)] = top_out; - stack_pointer[-1] = bottom_out; - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SWAP); + _PyStackRef bottom_in; + _PyStackRef top_in; + _PyStackRef top_out; + _PyStackRef bottom_out; + top_in = stack_pointer[-1]; + bottom_in = stack_pointer[-2 - (oparg-2)]; + bottom_out = bottom_in; + top_out = top_in; + assert(oparg >= 2); + stack_pointer[-2 - (oparg-2)] = top_out; + stack_pointer[-1] = bottom_out; + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL); - PREDICTED(TO_BOOL); - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - _PyStackRef value; - _PyStackRef res; - // _SPECIALIZE_TO_BOOL - { - value = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ToBool(value, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(TO_BOOL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _TO_BOOL - { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL); + PREDICTED(TO_BOOL); + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + _PyStackRef value; + _PyStackRef res; + // _SPECIALIZE_TO_BOOL + { + value = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + _Py_Specialize_ToBool(value, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (err < 0) CEVAL_GOTO(pop_1_error); - res = err ? PyStackRef_True : PyStackRef_False; + DISPATCH_SAME_OPARG(); } - stack_pointer[-1] = res; - DISPATCH(); + OPCODE_DEFERRED_INC(TO_BOOL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _TO_BOOL + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (err < 0) CEVAL_GOTO(pop_1_error); + res = err ? PyStackRef_True : PyStackRef_False; } + stack_pointer[-1] = res; + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL); - } - // _REPLACE_WITH_TRUE - { - value = owner; - PyStackRef_CLOSE(value); - res = PyStackRef_True; - } - stack_pointer[-1] = res; - DISPATCH(); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL); } + // _REPLACE_WITH_TRUE + { + value = owner; + PyStackRef_CLOSE(value); + res = PyStackRef_True; + } + stack_pointer[-1] = res; + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_BOOL); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_BOOL); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_INT); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL); - STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value_o)) { - assert(_Py_IsImmortal(value_o)); - res = PyStackRef_False; - } - else { - PyStackRef_CLOSE(value); - res = PyStackRef_True; - } - stack_pointer[-1] = res; - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_INT); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL); + STAT_INC(TO_BOOL, hit); + if (_PyLong_IsZero((PyLongObject *)value_o)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; + } + else { + PyStackRef_CLOSE(value); + res = PyStackRef_True; } + stack_pointer[-1] = res; + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_LIST); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL); - STAT_INC(TO_BOOL, hit); - res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; - PyStackRef_CLOSE(value); - stack_pointer[-1] = res; - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_LIST); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL); + STAT_INC(TO_BOOL, hit); + res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; + PyStackRef_CLOSE(value); + stack_pointer[-1] = res; + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_NONE); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - res = PyStackRef_False; - stack_pointer[-1] = res; - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_NONE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + // This one is a bit weird, because we expect *some* failures: + DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + res = PyStackRef_False; + stack_pointer[-1] = res; + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_STR); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL); - STAT_INC(TO_BOOL, hit); - if (value_o == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value_o)); - res = PyStackRef_False; - } - else { - assert(Py_SIZE(value_o)); - PyStackRef_CLOSE(value); - res = PyStackRef_True; - } - stack_pointer[-1] = res; - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_STR); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL); + STAT_INC(TO_BOOL, hit); + if (value_o == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; + } + else { + assert(Py_SIZE(value_o)); + PyStackRef_CLOSE(value); + res = PyStackRef_True; } + stack_pointer[-1] = res; + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_INVERT); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (res_o == NULL) CEVAL_GOTO(pop_1_error); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_INVERT); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) CEVAL_GOTO(pop_1_error); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-1] = res; + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NEGATIVE); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (res_o == NULL) CEVAL_GOTO(pop_1_error); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NEGATIVE); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) CEVAL_GOTO(pop_1_error); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-1] = res; + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NOT); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(value)); - res = PyStackRef_IsFalse(value) - ? PyStackRef_True : PyStackRef_False; - stack_pointer[-1] = res; - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NOT); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + stack_pointer[-1] = res; + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNPACK_EX); - _PyStackRef seq; - _PyStackRef *right; - seq = stack_pointer[-1]; - right = &stack_pointer[(oparg & 0xFF)]; - _PyStackRef *top = right + (oparg >> 8); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(seq); - if (res == 0) CEVAL_GOTO(pop_1_error); - stack_pointer += (oparg & 0xFF) + (oparg >> 8); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNPACK_EX); + _PyStackRef seq; + _PyStackRef *right; + seq = stack_pointer[-1]; + right = &stack_pointer[(oparg & 0xFF)]; + _PyStackRef *top = right + (oparg >> 8); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(seq); + if (res == 0) CEVAL_GOTO(pop_1_error); + stack_pointer += (oparg & 0xFF) + (oparg >> 8); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE); - PREDICTED(UNPACK_SEQUENCE); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef seq; - _PyStackRef *output; - // _SPECIALIZE_UNPACK_SEQUENCE - { - seq = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - (void)seq; - (void)counter; - } - // _UNPACK_SEQUENCE - { - output = &stack_pointer[-1]; - _PyStackRef *top = output + oparg; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE); + PREDICTED(UNPACK_SEQUENCE); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef seq; + _PyStackRef *output; + // _SPECIALIZE_UNPACK_SEQUENCE + { + seq = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(seq); - if (res == 0) CEVAL_GOTO(pop_1_error); + DISPATCH_SAME_OPARG(); } - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + (void)seq; + (void)counter; + } + // _UNPACK_SEQUENCE + { + output = &stack_pointer[-1]; + _PyStackRef *top = output + oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(seq); + if (res == 0) CEVAL_GOTO(pop_1_error); } + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef seq; - _PyStackRef *values; - /* Skip 1 cache entry */ - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE); - if (PyList_GET_SIZE(seq_o) != oparg) { - UNLOCK_OBJECT(seq_o); - DEOPT_IF(true, UNPACK_SEQUENCE); - } - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectNew(items[i]); - } + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef seq; + _PyStackRef *values; + /* Skip 1 cache entry */ + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE); + if (PyList_GET_SIZE(seq_o) != oparg) { UNLOCK_OBJECT(seq_o); - PyStackRef_CLOSE(seq); - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + DEOPT_IF(true, UNPACK_SEQUENCE); + } + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyList_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); } + UNLOCK_OBJECT(seq_o); + PyStackRef_CLOSE(seq); + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef seq; - _PyStackRef *values; - /* Skip 1 cache entry */ - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE); - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectNew(items[i]); - } - PyStackRef_CLOSE(seq); - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef seq; + _PyStackRef *values; + /* Skip 1 cache entry */ + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + PyStackRef_CLOSE(seq); + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef seq; - _PyStackRef val1; - _PyStackRef val0; - /* Skip 1 cache entry */ - seq = stack_pointer[-1]; - assert(oparg == 2); - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); - STAT_INC(UNPACK_SEQUENCE, hit); - val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); - val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - PyStackRef_CLOSE(seq); - stack_pointer[-1] = val1; - stack_pointer[0] = val0; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + /* Skip 1 cache entry */ + seq = stack_pointer[-1]; + assert(oparg == 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + PyStackRef_CLOSE(seq); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(WITH_EXCEPT_START); - _PyStackRef exit_func; - _PyStackRef exit_self; - _PyStackRef lasti; - _PyStackRef val; - _PyStackRef res; - val = stack_pointer[-1]; - lasti = stack_pointer[-3]; - exit_self = stack_pointer[-4]; - exit_func = stack_pointer[-5]; - /* At the top of the stack are 4 values: - - val: TOP = exc_info() - - unused: SECOND = previous exception - - lasti: THIRD = lasti of exception in exc_info() - - exit_self: FOURTH = the context or NULL - - exit_func: FIFTH = the context.__exit__ function or context.__exit__ bound method - We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). - Then we push the __exit__ return value. - */ - PyObject *exc, *tb; - PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); - PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); - assert(val_o && PyExceptionInstance_Check(val_o)); - exc = PyExceptionInstance_Class(val_o); - tb = PyException_GetTraceback(val_o); - if (tb == NULL) { - tb = Py_None; - } - else { - Py_DECREF(tb); - } - assert(PyStackRef_LongCheck(lasti)); - (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; - int has_self = !PyStackRef_IsNull(exit_self); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, - (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) CEVAL_GOTO(error); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(WITH_EXCEPT_START); + _PyStackRef exit_func; + _PyStackRef exit_self; + _PyStackRef lasti; + _PyStackRef val; + _PyStackRef res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_self = stack_pointer[-4]; + exit_func = stack_pointer[-5]; + /* At the top of the stack are 4 values: + - val: TOP = exc_info() + - unused: SECOND = previous exception + - lasti: THIRD = lasti of exception in exc_info() + - exit_self: FOURTH = the context or NULL + - exit_func: FIFTH = the context.__exit__ function or context.__exit__ bound method + We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). + Then we push the __exit__ return value. + */ + PyObject *exc, *tb; + PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); + PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); + assert(val_o && PyExceptionInstance_Check(val_o)); + exc = PyExceptionInstance_Class(val_o); + tb = PyException_GetTraceback(val_o); + if (tb == NULL) { + tb = Py_None; + } + else { + Py_DECREF(tb); + } + assert(PyStackRef_LongCheck(lasti)); + (void)lasti; // Shut up compiler warning if asserts are off + PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + int has_self = !PyStackRef_IsNull(exit_self); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) CEVAL_GOTO(error); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS){ - int opcode = next_instr->op.code; - (void)(opcode); - { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(YIELD_VALUE); - _PyStackRef retval; - _PyStackRef value; - retval = stack_pointer[-1]; - // NOTE: It's important that YIELD_VALUE never raises an exception! - // The compiler treats any exception raised here as a failed close() - // or throw() call. - #if TIER_ONE - assert(frame != entry_frame); - #endif - frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(YIELD_VALUE); + _PyStackRef retval; + _PyStackRef value; + retval = stack_pointer[-1]; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + #if TIER_ONE + assert(frame != entry_frame); + #endif + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - value = temp; - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + value = temp; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS){ -int opcode = next_instr->op.code; _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", _PyFrame_GetCode(frame)->co_filename, diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index def253a18a9007..a53f18f459fd28 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -50,8 +50,6 @@ def generate_label_handlers(outfile: TextIO): out.emit(function_proto(curr_proto)) out.emit("\n") out.emit("{\n") - out.emit("int opcode = next_instr->op.code;\n") - out.emit("(void)opcode;\n") for line in lines: if TARGET_LABEL in line: break @@ -93,19 +91,11 @@ def generate_tier1( out.emit("\n") out.emit(function_proto(name)) out.emit("{\n") - if analysis.opmap[name] >= analysis.min_instrumented: - out.emit(f"int opcode = {name};\n") - else: - out.emit("int opcode = next_instr->op.code;\n") - # Some instructions don't use opcode. - out.emit(f"(void)(opcode);\n") - out.emit("{\n") write_single_inst(out, emitter, name, inst) if not inst.parts[-1].properties.always_exits: out.emit("DISPATCH();\n") out.start_line() out.emit("}\n") - out.emit("}\n") out.emit("\n") @@ -113,7 +103,6 @@ def generate_tier1( out.emit(function_proto("UNKNOWN_OPCODE")) out.emit("{\n") out.emit(""" -int opcode = next_instr->op.code; _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", _PyFrame_GetCode(frame)->co_filename, From 56c6349826e1ef4dbc5f4699a81719671570059a Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 11 Jan 2025 01:40:27 +0800 Subject: [PATCH 027/110] Fix tests, move entry_frame into frame --- Include/internal/pycore_frame.h | 7 +++- Lib/test/test_generated_cases.py | 18 ++++----- Python/bytecodes.c | 9 ++--- Python/ceval.c | 54 ++++++++++++------------- Python/ceval_macros.h | 18 ++++----- Python/executor_cases.c.h | 4 +- Python/generated_cases.c.h | 12 +++--- Python/generated_tail_call_handlers.c.h | 22 +++++----- 8 files changed, 74 insertions(+), 70 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 96ae4dd22ecb43..ba11060c6c9617 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -75,7 +75,8 @@ typedef struct _PyInterpreterFrame { _PyStackRef *stackpointer; uint16_t return_offset; /* Only relevant during a function call */ char owner; - char visited; + char visited:4; + char is_entry_frame:4; /* Locals and stack */ _PyStackRef localsplus[1]; } _PyInterpreterFrame; @@ -214,6 +215,8 @@ _PyFrame_Initialize( frame->localsplus[i] = PyStackRef_NULL; } + frame->is_entry_frame = 0; + #ifdef Py_GIL_DISABLED // On GIL disabled, we walk the entire stack in GC. Since stacktop // is not always in sync with the real stack pointer, we have @@ -394,6 +397,8 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int frame->visited = 0; frame->return_offset = 0; + frame->is_entry_frame = 0; + #ifdef Py_GIL_DISABLED assert(code->co_nlocalsplus == 0); for (int i = 0; i < code->co_stacksize; i++) { diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 9c65e81dfe4be1..17e7b72a7818e1 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -538,7 +538,7 @@ def test_error_if_plain(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - if (cond) goto label; + if (cond) CEVAL_GOTO(label); DISPATCH(); } """ @@ -555,7 +555,7 @@ def test_error_if_plain_with_comment(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - if (cond) goto label; + if (cond) CEVAL_GOTO(label); // Comment is ok DISPATCH(); } @@ -582,7 +582,7 @@ def test_error_if_pop(self): right = stack_pointer[-1]; left = stack_pointer[-2]; SPAM(left, right); - if (cond) goto pop_2_label; + if (cond) CEVAL_GOTO(pop_2_label); res = 0; stack_pointer[-2] = res; stack_pointer += -1; @@ -611,7 +611,7 @@ def test_error_if_pop_with_result(self): right = stack_pointer[-1]; left = stack_pointer[-2]; res = SPAM(left, right); - if (cond) goto pop_2_label; + if (cond) CEVAL_GOTO(pop_2_label); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -929,7 +929,7 @@ def test_array_error_if(self): if (oparg == 0) { stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto somewhere; + CEVAL_GOTO(somewhere); } stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1392,7 +1392,7 @@ def test_pop_on_error_peeks(self): // THIRD { // Mark j and k as used - if (cond) goto pop_2_error; + if (cond) CEVAL_GOTO(pop_2_error); } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -1435,7 +1435,7 @@ def test_push_then_error(self): stack_pointer[1] = b; stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); - goto error; + CEVAL_GOTO(error); } } stack_pointer[0] = a; @@ -1462,14 +1462,14 @@ def test_error_if_true(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP1); - goto here; + CEVAL_GOTO(here); } TARGET(OP2) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP2); - goto there; + CEVAL_GOTO(there); } """ self.run_cases_test(input, output) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c17ca8099118b0..c0d01e86a5c9e4 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -101,7 +101,6 @@ dummy_func( PyObject *codeobj; PyObject *cond; PyObject *descr; - _PyInterpreterFrame entry_frame; PyObject *exc; PyObject *exit; PyObject *fget; @@ -1018,7 +1017,7 @@ dummy_func( } tier1 inst(INTERPRETER_EXIT, (retval --)) { - assert(frame == entry_frame); + assert(frame->is_entry_frame); assert(_PyFrame_IsIncomplete(frame)); /* Restore previous frame and return. */ tstate->current_frame = frame->previous; @@ -1034,7 +1033,7 @@ dummy_func( // is pushed to a different frame, the callers' frame. inst(RETURN_VALUE, (retval -- res)) { #if TIER_ONE - assert(frame != entry_frame); + assert(!frame->is_entry_frame); #endif _PyStackRef temp = retval; DEAD(retval); @@ -1133,7 +1132,7 @@ dummy_func( PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); PyObject *retval_o; - assert(frame != entry_frame); + assert(!frame->is_entry_frame); if ((tstate->interp->eval_frame == NULL) && (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) @@ -1207,7 +1206,7 @@ dummy_func( // The compiler treats any exception raised here as a failed close() // or throw() call. #if TIER_ONE - assert(frame != entry_frame); + assert(!frame->is_entry_frame); #endif frame->instr_ptr++; PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); diff --git a/Python/ceval.c b/Python/ceval.c index 4d3414e3d99018..adfb04ef69e442 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -229,12 +229,12 @@ lltrace_resume_frame(_PyInterpreterFrame *frame) } static int -maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, _PyInterpreterFrame *skip_frame, PyObject *globals) +maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, PyObject *globals) { if (globals == NULL) { return 0; } - if (frame == skip_frame) { + if (frame->is_entry_frame) { return 0; } int r = PyDict_Contains(globals, &_Py_ID(__lltrace__)); @@ -787,9 +787,9 @@ static inline PyObject *_TAIL_CALL_shim(TAIL_CALL_PARAMS) { opcode = next_instr->op.code; #ifdef LLTRACE - return (INSTRUCTION_TABLE[opcode])(frame, stack_pointer, tstate, next_instr, opcode, next_instr->op.arg, entry_frame, lltrace); + return (INSTRUCTION_TABLE[opcode])(frame, stack_pointer, tstate, next_instr, opcode, next_instr->op.arg, lltrace); #else - return (INSTRUCTION_TABLE[opcode])(frame, stack_pointer, tstate, next_instr, opcode, next_instr->op.arg, entry_frame); + return (INSTRUCTION_TABLE[opcode])(frame, stack_pointer, tstate, next_instr, opcode, next_instr->op.arg); #endif } #endif @@ -816,28 +816,28 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int int lltrace = 0; #endif - _PyInterpreterFrame entry_f; - _PyInterpreterFrame *entry_frame = &entry_f; + _PyInterpreterFrame entry_frame; #if defined(Py_DEBUG) && !defined(Py_STACKREF_DEBUG) /* Set these to invalid but identifiable values for debugging. */ - entry_f.f_funcobj = (_PyStackRef){.bits = 0xaaa0}; - entry_f.f_locals = (PyObject*)0xaaa1; - entry_f.frame_obj = (PyFrameObject*)0xaaa2; - entry_f.f_globals = (PyObject*)0xaaa3; - entry_f.f_builtins = (PyObject*)0xaaa4; + entry_frame.f_funcobj = (_PyStackRef){.bits = 0xaaa0}; + entry_frame.f_locals = (PyObject*)0xaaa1; + entry_frame.frame_obj = (PyFrameObject*)0xaaa2; + entry_frame.f_globals = (PyObject*)0xaaa3; + entry_frame.f_builtins = (PyObject*)0xaaa4; #endif - entry_f.f_executable = PyStackRef_None; - entry_f.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; - entry_f.stackpointer = entry_f.localsplus; - entry_f.owner = FRAME_OWNED_BY_CSTACK; - entry_f.visited = 0; - entry_f.return_offset = 0; + entry_frame.f_executable = PyStackRef_None; + entry_frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; + entry_frame.stackpointer = entry_frame.localsplus; + entry_frame.owner = FRAME_OWNED_BY_CSTACK; + entry_frame.visited = 0; + entry_frame.return_offset = 0; /* Push frame */ - entry_f.previous = tstate->current_frame; - frame->previous = entry_frame; + entry_frame.previous = tstate->current_frame; + entry_frame.is_entry_frame = 1; + frame->previous = &entry_frame; tstate->current_frame = frame; tstate->c_recursion_remaining -= (PY_EVAL_C_STACK_UNITS - 1); @@ -894,7 +894,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int stack_pointer = _PyFrame_GetStackPointer(frame); #ifdef LLTRACE - lltrace = maybe_lltrace_resume_frame(frame, entry_frame, GLOBALS()); + lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } @@ -909,9 +909,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #ifdef Py_TAIL_CALL_INTERP #ifdef LLTRACE - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0, entry_frame, lltrace); + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0, lltrace); #else - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0, entry_frame); + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0); #endif #else DISPATCH(); @@ -968,7 +968,7 @@ TAIL_CALL_TARGET(error): #endif /* Log traceback info. */ - assert(frame != entry_frame); + assert(!frame->is_entry_frame); if (!_PyFrame_IsIncomplete(frame)) { PyFrameObject *f = _PyFrame_GetFrameObject(frame); if (f != NULL) { @@ -1033,9 +1033,9 @@ TAIL_CALL_TARGET(exception_unwind): DISPATCH(); # else # ifdef LLTRACE - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0, entry_frame, lltrace); + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0, lltrace); # else - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0, entry_frame); + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0); # endif # endif #else @@ -1046,13 +1046,13 @@ TAIL_CALL_TARGET(exception_unwind): TAIL_CALL_TARGET(exit_unwind): assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); + assert(!frame->is_entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; - if (frame == entry_frame) { + if (frame->is_entry_frame) { /* Restore previous frame and exit */ tstate->current_frame = frame->previous; tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 39e22768c2d720..dabbb16d7185ab 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -71,11 +71,11 @@ #endif #ifdef LLTRACE -# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int opcode, int oparg, _PyInterpreterFrame *entry_frame, int lltrace -# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, opcode, oparg, entry_frame, lltrace +# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int opcode, int oparg, int lltrace +# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, opcode, oparg, lltrace #else -# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int opcode, int oparg, _PyInterpreterFrame *entry_frame -# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, opcode, oparg, entry_frame +# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int opcode, int oparg +# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, opcode, oparg #endif #ifdef Py_TAIL_CALL_INTERP @@ -116,7 +116,7 @@ #if LLTRACE #define LLTRACE_RESUME_FRAME() \ do { \ - lltrace = maybe_lltrace_resume_frame(frame, entry_frame, GLOBALS()); \ + lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); \ if (lltrace < 0) { \ CEVAL_GOTO(exit_unwind); \ } \ @@ -162,7 +162,7 @@ do { \ } \ next_instr = frame->instr_ptr; \ stack_pointer = _PyFrame_GetStackPointer(frame); \ - lltrace = maybe_lltrace_resume_frame(frame, entry_frame, GLOBALS()); \ + lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); \ if (lltrace < 0) { \ CEVAL_GOTO(exit_unwind); \ } \ @@ -307,7 +307,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { #endif #define WITHIN_STACK_BOUNDS() \ - (frame == entry_frame || (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE())) + (frame->is_entry_frame || (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE())) /* Data access macros */ #define FRAME_CO_CONSTS (_PyFrame_GetCode(frame)->co_consts) @@ -331,12 +331,12 @@ GETITEM(PyObject *v, Py_ssize_t i) { #ifdef LLTRACE #define GO_TO_INSTRUCTION(op) do { \ Py_MUSTTAIL \ - return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - _PyOpcode_Caches[_PyOpcode_Deopt[op]], opcode, oparg, entry_frame, lltrace); \ + return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - _PyOpcode_Caches[_PyOpcode_Deopt[op]], opcode, oparg, lltrace); \ } while (0) #else #define GO_TO_INSTRUCTION(op) do { \ Py_MUSTTAIL \ - return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - _PyOpcode_Caches[_PyOpcode_Deopt[op]], opcode, oparg, entry_frame); \ + return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - _PyOpcode_Caches[_PyOpcode_Deopt[op]], opcode, oparg); \ } while (0) #endif #else diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 026adafe264092..e4dfcf84dd950e 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1362,7 +1362,7 @@ _PyStackRef res; retval = stack_pointer[-1]; #if TIER_ONE - assert(frame != entry_frame); + assert(!frame->is_entry_frame); #endif _PyStackRef temp = retval; stack_pointer += -1; @@ -1500,7 +1500,7 @@ // The compiler treats any exception raised here as a failed close() // or throw() call. #if TIER_ONE - assert(frame != entry_frame); + assert(!frame->is_entry_frame); #endif frame->instr_ptr++; PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2a3657df54a304..7fc770c151e403 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4936,7 +4936,7 @@ { retval = val; #if TIER_ONE - assert(frame != entry_frame); + assert(!frame->is_entry_frame); #endif _PyStackRef temp = retval; stack_pointer += -1; @@ -4990,7 +4990,7 @@ // The compiler treats any exception raised here as a failed close() // or throw() call. #if TIER_ONE - assert(frame != entry_frame); + assert(!frame->is_entry_frame); #endif frame->instr_ptr++; PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); @@ -5034,7 +5034,7 @@ INSTRUCTION_STATS(INTERPRETER_EXIT); _PyStackRef retval; retval = stack_pointer[-1]; - assert(frame == entry_frame); + assert(frame->is_entry_frame); assert(_PyFrame_IsIncomplete(frame)); /* Restore previous frame and return. */ tstate->current_frame = frame->previous; @@ -7055,7 +7055,7 @@ _PyStackRef res; retval = stack_pointer[-1]; #if TIER_ONE - assert(frame != entry_frame); + assert(!frame->is_entry_frame); #endif _PyStackRef temp = retval; stack_pointer += -1; @@ -7109,7 +7109,7 @@ v = stack_pointer[-1]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); PyObject *retval_o; - assert(frame != entry_frame); + assert(!frame->is_entry_frame); if ((tstate->interp->eval_frame == NULL) && (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) @@ -8220,7 +8220,7 @@ // The compiler treats any exception raised here as a failed close() // or throw() call. #if TIER_ONE - assert(frame != entry_frame); + assert(!frame->is_entry_frame); #endif frame->instr_ptr++; PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 26a6a481464190..fc2a0c6f82a66a 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -51,7 +51,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS) #endif /* Log traceback info. */ - assert(frame != entry_frame); + assert(!frame->is_entry_frame); if (!_PyFrame_IsIncomplete(frame)) { PyFrameObject *f = _PyFrame_GetFrameObject(frame); if (f != NULL) { @@ -119,9 +119,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAM DISPATCH(); # else # ifdef LLTRACE - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0, entry_frame, lltrace); + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0, lltrace); # else - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0, entry_frame); + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0); # endif # endif #else @@ -135,13 +135,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS) { assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); - assert(frame != entry_frame); + assert(!frame->is_entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; - if (frame == entry_frame) { + if (frame->is_entry_frame) { /* Restore previous frame and exit */ tstate->current_frame = frame->previous; tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; @@ -5087,7 +5087,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_C { retval = val; #if TIER_ONE - assert(frame != entry_frame); + assert(!frame->is_entry_frame); #endif _PyStackRef temp = retval; stack_pointer += -1; @@ -5141,7 +5141,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CA // The compiler treats any exception raised here as a failed close() // or throw() call. #if TIER_ONE - assert(frame != entry_frame); + assert(!frame->is_entry_frame); #endif frame->instr_ptr++; PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); @@ -5185,7 +5185,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAM INSTRUCTION_STATS(INTERPRETER_EXIT); _PyStackRef retval; retval = stack_pointer[-1]; - assert(frame == entry_frame); + assert(frame->is_entry_frame); assert(_PyFrame_IsIncomplete(frame)); /* Restore previous frame and return. */ tstate->current_frame = frame->previous; @@ -7206,7 +7206,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS){ _PyStackRef res; retval = stack_pointer[-1]; #if TIER_ONE - assert(frame != entry_frame); + assert(!frame->is_entry_frame); #endif _PyStackRef temp = retval; stack_pointer += -1; @@ -7260,7 +7260,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS){ v = stack_pointer[-1]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); PyObject *retval_o; - assert(frame != entry_frame); + assert(!frame->is_entry_frame); if ((tstate->interp->eval_frame == NULL) && (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) @@ -8371,7 +8371,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS){ // The compiler treats any exception raised here as a failed close() // or throw() call. #if TIER_ONE - assert(frame != entry_frame); + assert(!frame->is_entry_frame); #endif frame->instr_ptr++; PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); From 34dd801edd5c2f8a3b4e4a0eeaab9c6886cbf81b Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 11 Jan 2025 02:01:55 +0800 Subject: [PATCH 028/110] Block tier2 --- configure | 6 ++++-- configure.ac | 10 ++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/configure b/configure index 48ed56b860ae92..380d9bb377e033 100755 --- a/configure +++ b/configure @@ -29301,11 +29301,13 @@ esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_tail_call" >&5 printf "%s\n" "$ac_cv_tail_call" >&6; } -case "$ac_cv_tail_call" in yes*) +if $tier2_flags == ""; then + case "$ac_cv_tail_call" in yes*) printf "%s\n" "#define Py_TAIL_CALL_INTERP 1" >>confdefs.h -esac + esac +fi case $ac_sys_system in AIX*) diff --git a/configure.ac b/configure.ac index f2d4912322b28b..7a45bd9fb847a5 100644 --- a/configure.ac +++ b/configure.ac @@ -7046,10 +7046,12 @@ AC_COMPILE_IFELSE([AC_LANG_SOURCE([[[ else ac_cv_tail_call=no fi])) -case "$ac_cv_tail_call" in yes*) - AC_DEFINE([Py_TAIL_CALL_INTERP], [1], - [Define if the C compiler supports efficient proper tail calls.]) -esac +if $tier2_flags == ""; then + case "$ac_cv_tail_call" in yes*) + AC_DEFINE([Py_TAIL_CALL_INTERP], [1], + [Define if the C compiler supports efficient proper tail calls.]) + esac +fi case $ac_sys_system in AIX*) From aa674277a0b1545bd99c6c9d5ba7510c59ee5f11 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 11 Jan 2025 02:32:12 +0800 Subject: [PATCH 029/110] Add tests in cases generator --- Lib/test/test_generated_cases.py | 152 +++++++++++++ Python/generated_tail_call_handlers.c.h | 205 +++++++++--------- .../tier1_tail_call_generator.py | 24 +- 3 files changed, 276 insertions(+), 105 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 17e7b72a7818e1..92d78ec4107d3f 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -4,6 +4,7 @@ import sys import tempfile import unittest +import textwrap from io import StringIO from test import support @@ -36,6 +37,7 @@ def skip_if_different_mount_drives(): import parser from stack import Local, Stack import tier1_generator + import tier1_tail_call_generator import opcode_metadata_generator import optimizer_generator @@ -1714,6 +1716,156 @@ def test_pop_dead_inputs_with_output(self): self.run_cases_test(input, output) +class TestGeneratedTailCallErorHandlers(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.maxDiff = None + + self.temp_dir = tempfile.gettempdir() + self.temp_input_filename = os.path.join(self.temp_dir, "input.txt") + self.temp_output_filename = os.path.join(self.temp_dir, "output.txt") + self.temp_metadata_filename = os.path.join(self.temp_dir, "metadata.txt") + self.temp_pymetadata_filename = os.path.join(self.temp_dir, "pymetadata.txt") + self.temp_executor_filename = os.path.join(self.temp_dir, "executor.txt") + + def tearDown(self) -> None: + for filename in [ + self.temp_input_filename, + self.temp_output_filename, + self.temp_metadata_filename, + self.temp_pymetadata_filename, + self.temp_executor_filename, + ]: + try: + os.remove(filename) + except: + pass + super().tearDown() + + def run_cases_test(self, input: str, expected: str): + with open(self.temp_input_filename, "w+") as temp_input: + temp_input.write(textwrap.dedent(input)) + temp_input.flush() + + with handle_stderr(): + tier1_tail_call_generator.generate_label_handlers_from_files( + self.temp_input_filename, self.temp_output_filename + ) + + with open(self.temp_output_filename) as temp_output: + lines = temp_output.readlines() + while lines and lines[0].startswith(("// ", "#", " #", "\n")): + lines.pop(0) + while lines and lines[-1].startswith(("#", "\n")): + lines.pop(-1) + actual = "".join(lines) + + self.assertEqual(actual.strip(), textwrap.dedent(expected).strip()) + + def test_correctly_finds_pyeval_framedefault(self): + input = """ + PyObject* _Py_HOT_FUNCTION + _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) + { + + /* END_BASE_INTERPRETER */ + } + """ + output = """ + """ + self.run_cases_test(input, output) + + def test_simple_label(self): + input = """ + PyObject* _Py_HOT_FUNCTION + _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) + { + + TAIL_CALL_TARGET(error): + DO_THING(); + /* END_BASE_INTERPRETER */ + } + """ + output = """ + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); + + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS) + { + DO_THING(); + /* END_BASE_INTERPRETER */ + } + """ + self.run_cases_test(input, output) + + def test_fallthrough_label(self): + input = """ + PyObject* _Py_HOT_FUNCTION + _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) + { + + TAIL_CALL_TARGET(error): + DO_THING(); + TAIL_CALL_TARGET(fallthrough): + DO_THING2(); + /* END_BASE_INTERPRETER */ + } + """ + output = """ + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_fallthrough(TAIL_CALL_PARAMS); + + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS) + { + DO_THING(); + CEVAL_GOTO(fallthrough); + } + + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_fallthrough(TAIL_CALL_PARAMS) + { + DO_THING2(); + /* END_BASE_INTERPRETER */ + } + """ + self.run_cases_test(input, output) + + def test_transform_gotos(self): + input = """ + PyObject* _Py_HOT_FUNCTION + _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) + { + + TAIL_CALL_TARGET(error): + if (thing) { + goto fallthrough; + } + DO_THING(); + TAIL_CALL_TARGET(fallthrough): + DO_THING2(); + /* END_BASE_INTERPRETER */ + } + """ + output = """ + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_fallthrough(TAIL_CALL_PARAMS); + + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS) + { + if (thing) { + CEVAL_GOTO(fallthrough); + } + DO_THING(); + CEVAL_GOTO(fallthrough); + } + + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_fallthrough(TAIL_CALL_PARAMS) + { + DO_THING2(); + /* END_BASE_INTERPRETER */ + } + """ + self.run_cases_test(input, output) + + class TestGeneratedAbstractCases(unittest.TestCase): def setUp(self) -> None: super().setUp() diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index fc2a0c6f82a66a..87cd81edcb72b9 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -18,148 +18,157 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_resume_with_error(TAIL_CALL_PARAMS); + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_4_error(TAIL_CALL_PARAMS) { - STACK_SHRINK(1); + STACK_SHRINK(1); CEVAL_GOTO(pop_3_error); } + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_3_error(TAIL_CALL_PARAMS) { - STACK_SHRINK(1); + STACK_SHRINK(1); CEVAL_GOTO(pop_2_error); } + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS) { - STACK_SHRINK(1); + STACK_SHRINK(1); CEVAL_GOTO(pop_1_error); } + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS) { - STACK_SHRINK(1); + STACK_SHRINK(1); CEVAL_GOTO(error); } + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS) { - /* Double-check exception status. */ - #ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } - #else - assert(_PyErr_Occurred(tstate)); - #endif + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif - /* Log traceback info. */ - assert(!frame->is_entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } + /* Log traceback info. */ + assert(!frame->is_entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); CEVAL_GOTO(exception_unwind); } + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS) { - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - CEVAL_GOTO(exit_unwind); - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - CEVAL_GOTO(exception_unwind); - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + CEVAL_GOTO(exit_unwind); + } - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + CEVAL_GOTO(exception_unwind); + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - CEVAL_GOTO(exception_unwind); - } - /* Resume normal execution */ - #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } - #endif + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; - #ifdef Py_TAIL_CALL_INTERP - # ifdef IN_TAIL_CALL_INTERP - DISPATCH(); - # else - # ifdef LLTRACE - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0, lltrace); - # else - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0); - # endif - # endif - #else - DISPATCH(); - #endif + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + CEVAL_GOTO(exception_unwind); } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + +#ifdef Py_TAIL_CALL_INTERP +# ifdef IN_TAIL_CALL_INTERP + DISPATCH(); +# else +# ifdef LLTRACE + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0, lltrace); +# else + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0); +# endif +# endif +#else + DISPATCH(); +#endif + } CEVAL_GOTO(exit_unwind); } + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS) { - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(!frame->is_entry_frame); + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(!frame->is_entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame->is_entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame->is_entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } CEVAL_GOTO(resume_with_error); } + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_resume_with_error(TAIL_CALL_PARAMS) { - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); CEVAL_GOTO(error); - /* END_BASE_INTERPRETER */ +/* END_BASE_INTERPRETER */ } + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 2; diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index a53f18f459fd28..6593e043267633 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -33,15 +33,15 @@ TARGET_LABEL = "TAIL_CALL_TARGET" -def generate_label_handlers(outfile: TextIO): +def generate_label_handlers(infile: TextIO, outfile: TextIO): out = CWriter(outfile, 0, False) - with open(DEFAULT_CEVAL_INPUT, "r") as fp: - str_in = fp.read() - # https://stackoverflow.com/questions/8303488/regex-to-match-any-character-including-new-lines - eval_framedefault = re.findall("_PyEval_EvalFrameDefault\(.*\)\n({[\s\S]*\/\* END_BASE_INTERPRETER \*\/)", str_in)[0] + str_in = infile.read() + # https://stackoverflow.com/questions/8303488/regex-to-match-any-character-including-new-lines + eval_framedefault = re.findall("_PyEval_EvalFrameDefault\(.*\)\n({[\s\S]*\/\* END_BASE_INTERPRETER \*\/)", str_in)[0] function_protos = re.findall(f"{TARGET_LABEL}\((\w+)\):", eval_framedefault) for proto in function_protos: out.emit(f"{function_proto(proto)};\n") + out.emit("\n") lines = iter(eval_framedefault[eval_framedefault.find(TARGET_LABEL):].split("\n")) next(lines) for i in range(len(function_protos)): @@ -56,11 +56,20 @@ def generate_label_handlers(outfile: TextIO): if label := re.findall("goto (\w+);", line): out.emit(f"CEVAL_GOTO({label[0]});\n") else: - out.emit(line) + out.emit_text(line) out.emit("\n") if fallthrough_proto: out.emit(f"CEVAL_GOTO({fallthrough_proto});\n") out.emit("}\n") + out.emit("\n") + +# For unit testing. +def generate_label_handlers_from_files( + infilename: str, outfilename: str +) -> None: + with open(infilename, "r") as infile, open(outfilename, "w") as outfile: + generate_label_handlers(infile, outfile) + def function_proto(name: str) -> str: @@ -83,7 +92,8 @@ def generate_tier1( out.emit("static inline PyObject *_TAIL_CALL_shim(TAIL_CALL_PARAMS);\n") out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256];\n"); - generate_label_handlers(outfile) + with open(DEFAULT_CEVAL_INPUT, "r") as infile: + generate_label_handlers(infile, outfile) emitter = Emitter(out) out.emit("\n") From 54cff797dc72546d7149a437d0d1f71fb99c6f3b Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 11 Jan 2025 02:37:22 +0800 Subject: [PATCH 030/110] Leave assert in --- Python/ceval_macros.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index dabbb16d7185ab..bb6f2d07277f3e 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -361,7 +361,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { if ((COND)) { \ /* This is only a single jump on release builds! */ \ UPDATE_MISS_STATS((INSTNAME)); \ - /* assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); */ \ + assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); \ GO_TO_INSTRUCTION(INSTNAME); \ } From 75361d4041e8637b13b12e83381d5adaaf40edb0 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 11 Jan 2025 02:52:14 +0800 Subject: [PATCH 031/110] Upstream changes --- Python/generated_tail_call_handlers.c.h | 93 +++++++++++++++++------- Tools/cases_generator/tier1_generator.py | 7 +- 2 files changed, 71 insertions(+), 29 deletions(-) diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 87cd81edcb72b9..6473b01782912c 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -240,6 +240,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PA { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left_o)->ob_fval + @@ -276,6 +278,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARA { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); @@ -311,6 +315,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_ { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = PyUnicode_Concat(left_o, right_o); PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); @@ -345,6 +351,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TA { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); int next_oparg; #if TIER_ONE assert(next_instr->op.code == STORE_FAST); @@ -407,6 +415,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CA { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left_o)->ob_fval * @@ -443,6 +453,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); @@ -478,6 +490,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CA { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left_o)->ob_fval - @@ -514,6 +528,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); @@ -3393,7 +3409,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS){ left = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3932,11 +3948,15 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS){ } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_FOR); _PyStackRef value; value = stack_pointer[-1]; + /* Don't update instr_ptr, so that POP_ITER sees + * the FOR_ITER as the previous instruction. + * This has the benign side effect that if value is + * finalized it will see the location as the FOR_ITER's. + */ PyStackRef_CLOSE(value); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -4121,10 +4141,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS){ /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - PyStackRef_CLOSE(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */ - JUMPBY(oparg + 2); + /* Jump forward oparg, then skip following END_FOR */ + JUMPBY(oparg + 1); DISPATCH(); } next = PyStackRef_FromPyObjectSteal(next_o); @@ -4212,10 +4230,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS){ Py_DECREF(seq); } #endif - PyStackRef_CLOSE(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ - JUMPBY(oparg + 2); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); DISPATCH(); } } @@ -4255,10 +4271,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS) assert(Py_TYPE(r) == &PyRangeIter_Type); STAT_INC(FOR_ITER, hit); if (r->len <= 0) { - STACK_SHRINK(1); - PyStackRef_CLOSE(iter); - // Jump over END_FOR and POP_TOP instructions. - JUMPBY(oparg + 2); + // Jump over END_FOR instruction. + JUMPBY(oparg + 1); DISPATCH(); } } @@ -4305,10 +4319,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS) it->it_seq = NULL; Py_DECREF(seq); } - PyStackRef_CLOSE(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ - JUMPBY(oparg + 2); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); DISPATCH(); } } @@ -4736,7 +4748,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_P } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_FOR); @@ -4800,6 +4812,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_ stack_pointer = _PyFrame_GetStackPointer(frame); if (next != NULL) { PUSH(PyStackRef_FromPyObjectSteal(next)); + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); } else { if (_PyErr_Occurred(tstate)) { @@ -4817,11 +4830,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_ /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - STACK_SHRINK(1); - PyStackRef_CLOSE(iter_stackref); - /* Skip END_FOR and POP_TOP */ - _Py_CODEUNIT *target = next_instr + oparg + 2; - INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH_RIGHT); + /* Skip END_FOR */ + JUMPBY(oparg + 1); } DISPATCH(); } @@ -4928,11 +4938,28 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAI } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS){ + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); - INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + (void)this_instr; // INSTRUMENTED_JUMP requires this_instr + INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + DISPATCH(); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS){ + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); + _PyStackRef iter; + iter = stack_pointer[-1]; + INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); + PyStackRef_CLOSE(iter); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6857,6 +6884,18 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS){ DISPATCH(); } +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_ITER); + _PyStackRef value; + value = stack_pointer[-1]; + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); +} + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS){ _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -8531,6 +8570,7 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [INSTRUMENTED_LINE] = _TAIL_CALL_INSTRUMENTED_LINE, [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR, [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_INSTRUMENTED_NOT_TAKEN, + [INSTRUMENTED_POP_ITER] = _TAIL_CALL_INSTRUMENTED_POP_ITER, [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE, [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, @@ -8590,6 +8630,7 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [NOP] = _TAIL_CALL_NOP, [NOT_TAKEN] = _TAIL_CALL_NOT_TAKEN, [POP_EXCEPT] = _TAIL_CALL_POP_EXCEPT, + [POP_ITER] = _TAIL_CALL_POP_ITER, [POP_JUMP_IF_FALSE] = _TAIL_CALL_POP_JUMP_IF_FALSE, [POP_JUMP_IF_NONE] = _TAIL_CALL_POP_JUMP_IF_NONE, [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_POP_JUMP_IF_NOT_NONE, @@ -8642,7 +8683,6 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, - [117] = _TAIL_CALL_UNKNOWN_OPCODE, [118] = _TAIL_CALL_UNKNOWN_OPCODE, [119] = _TAIL_CALL_UNKNOWN_OPCODE, [120] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -8681,7 +8721,6 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [232] = _TAIL_CALL_UNKNOWN_OPCODE, [233] = _TAIL_CALL_UNKNOWN_OPCODE, [234] = _TAIL_CALL_UNKNOWN_OPCODE, - [235] = _TAIL_CALL_UNKNOWN_OPCODE, }; #undef TIER_ONE #undef IN_TAIL_CALL_INTERP diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 8763eb02b1d107..5677664f894735 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -134,9 +134,12 @@ def write_single_inst(out: CWriter, emitter: Emitter, name: str, inst: Instructi if inst.properties.needs_prev: out.emit(f"_Py_CODEUNIT* const prev_instr = frame->instr_ptr;\n") if needs_this and not inst.is_target: - out.emit(f"_Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;\n") + if inst.properties.no_save_ip: + out.emit(f"_Py_CODEUNIT* const this_instr = next_instr;\n") + else: + out.emit(f"_Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;\n") out.emit(unused_guard) - else: + elif not inst.properties.no_save_ip: out.emit(f"frame->instr_ptr = next_instr;\n") out.emit(f"next_instr += {inst.size};\n") out.emit(f"INSTRUCTION_STATS({name});\n") From d93bdaa8faa4681f6271a7f036a124b7c5ba86f2 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Fri, 10 Jan 2025 18:56:25 +0000 Subject: [PATCH 032/110] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by?= =?UTF-8?q?=20blurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2025-01-10-18-56-20.gh-issue-128563.baDvls.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst new file mode 100644 index 00000000000000..3e942358016411 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst @@ -0,0 +1 @@ +A new type of interpreter has been added to CPython. This interpreter uses tail calls for its instruction handlers. Preliminary benchmark results suggest 7.4-14.7% geometric mean faster on pyperformance (depending on platform), and up to 45% faster on Python-intensive workloads. Patch by Ken Jin, with ideas by Garret Gu, Haoran Xu, and Josh Haberman. From a68d891ffe7cc1c1da757365d346b27ff624a480 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 11 Jan 2025 02:57:22 +0800 Subject: [PATCH 033/110] fix mypy --- Tools/cases_generator/tier1_generator.py | 2 +- Tools/cases_generator/tier1_tail_call_generator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 5677664f894735..d0b2781e982ca8 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -128,7 +128,7 @@ def uses_this(inst: Instruction) -> bool: return False -def write_single_inst(out: CWriter, emitter: Emitter, name: str, inst: Instruction): +def write_single_inst(out: CWriter, emitter: Emitter, name: str, inst: Instruction) -> None: needs_this = uses_this(inst) unused_guard = "(void)this_instr;\n" if inst.family is None else "" if inst.properties.needs_prev: diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index 6593e043267633..da259273d5378e 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -33,7 +33,7 @@ TARGET_LABEL = "TAIL_CALL_TARGET" -def generate_label_handlers(infile: TextIO, outfile: TextIO): +def generate_label_handlers(infile: TextIO, outfile: TextIO) -> None: out = CWriter(outfile, 0, False) str_in = infile.read() # https://stackoverflow.com/questions/8303488/regex-to-match-any-character-including-new-lines From 1310b47e7f0aa1aaeaa5ca04094d0fbf6597232e Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 11 Jan 2025 03:00:13 +0800 Subject: [PATCH 034/110] Fix lint --- Lib/test/test_generated_cases.py | 8 ++++---- Python/generated_tail_call_handlers.c.h | 2 +- Tools/cases_generator/tier1_tail_call_generator.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 014357513a543c..00161f9f75d3bb 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1792,7 +1792,7 @@ def test_correctly_finds_pyeval_framedefault(self): PyObject* _Py_HOT_FUNCTION _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { - + /* END_BASE_INTERPRETER */ } """ @@ -1805,7 +1805,7 @@ def test_simple_label(self): PyObject* _Py_HOT_FUNCTION _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { - + TAIL_CALL_TARGET(error): DO_THING(); /* END_BASE_INTERPRETER */ @@ -1827,7 +1827,7 @@ def test_fallthrough_label(self): PyObject* _Py_HOT_FUNCTION _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { - + TAIL_CALL_TARGET(error): DO_THING(); TAIL_CALL_TARGET(fallthrough): @@ -1858,7 +1858,7 @@ def test_transform_gotos(self): PyObject* _Py_HOT_FUNCTION _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { - + TAIL_CALL_TARGET(error): if (thing) { goto fallthrough; diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 6473b01782912c..b3e71af5c701b1 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -8462,7 +8462,7 @@ _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", _PyFrame_GetCode(frame)->co_filename, PyUnstable_InterpreterFrame_GetLine(frame), - opcode); + opcode); CEVAL_GOTO(error);} static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [BINARY_OP] = _TAIL_CALL_BINARY_OP, diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index da259273d5378e..30e070c22acb42 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -117,7 +117,7 @@ def generate_tier1( "%U:%d: unknown opcode %d", _PyFrame_GetCode(frame)->co_filename, PyUnstable_InterpreterFrame_GetLine(frame), - opcode); + opcode); """) out.emit("CEVAL_GOTO(error);") out.emit("}\n") From db110fff45fc612694f8a9fc34ace58686105f8c Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 11 Jan 2025 03:28:53 +0800 Subject: [PATCH 035/110] Fix builds, address review --- .gitattributes | 1 + .github/workflows/tail-call.yml | 111 ++++++++++++++++++++++++++++ Tools/c-analyzer/cpython/_parser.py | 1 + Tools/jit/template.c | 7 ++ 4 files changed, 120 insertions(+) create mode 100644 .github/workflows/tail-call.yml diff --git a/.gitattributes b/.gitattributes index 2f5a030981fb94..ff479c49ba5394 100644 --- a/.gitattributes +++ b/.gitattributes @@ -95,6 +95,7 @@ Programs/test_frozenmain.h generated Python/Python-ast.c generated Python/executor_cases.c.h generated Python/generated_cases.c.h generated +Python/generated_tail_call_handlers.c.h generated Python/optimizer_cases.c.h generated Python/opcode_targets.h generated Python/stdlib_module_names.h generated diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml new file mode 100644 index 00000000000000..6920fd3e7348a3 --- /dev/null +++ b/.github/workflows/tail-call.yml @@ -0,0 +1,111 @@ +name: JIT +on: + pull_request: + paths: + - 'Python/bytecodes.c' + - 'Python/generated_tail_call_handlers.c.h' + push: + paths: + - 'Python/bytecodes.c' + - 'Python/generated_tail_call_handlers.c.h' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + jit: + name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + target: +# Un-comment as we add support for more platforms for tail-calling interpreters. +# - i686-pc-windows-msvc/msvc +# - x86_64-pc-windows-msvc/msvc +# - aarch64-pc-windows-msvc/msvc +# - x86_64-apple-darwin/clang +# - aarch64-apple-darwin/clang + - x86_64-unknown-linux-gnu/gcc +# - aarch64-unknown-linux-gnu/gcc + debug: + - true +# - false + llvm: + - 19 + include: + - target: i686-pc-windows-msvc/msvc + architecture: Win32 + runner: windows-latest + - target: x86_64-pc-windows-msvc/msvc + architecture: x64 + runner: windows-latest + - target: aarch64-pc-windows-msvc/msvc + architecture: ARM64 + runner: windows-latest + - target: x86_64-apple-darwin/clang + architecture: x86_64 + runner: macos-13 + - target: aarch64-apple-darwin/clang + architecture: aarch64 + runner: macos-14 + - target: x86_64-unknown-linux-gnu/gcc + architecture: x86_64 + runner: ubuntu-24.04 + - target: aarch64-unknown-linux-gnu/gcc + architecture: aarch64 + # Forks don't have access to our paid AArch64 runners. These jobs are skipped below: + runner: ${{ github.repository_owner == 'python' && 'ubuntu-24.04-aarch64' || 'ubuntu-24.04' }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Native Windows + if: runner.os == 'Windows' && matrix.architecture != 'ARM64' + run: | + choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 + ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} + ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + # No tests (yet): + - name: Emulated Windows + if: runner.os == 'Windows' && matrix.architecture == 'ARM64' + run: | + choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 + ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} + + # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. + # This is a bug in the macOS runner image where the pre-installed Python is installed in the same + # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes + # the symlink to the pre-installed Python so that the Homebrew Python is used instead. + - name: Native macOS + if: runner.os == 'macOS' + run: | + brew update + find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete + brew install llvm@${{ matrix.llvm }} + export SDKROOT="$(xcrun --show-sdk-path)" + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} + make all --jobs 4 + ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + - name: Native Linux + # Forks don't have access to our paid AArch64 runners. Skip those: + if: runner.os == 'Linux' && (matrix.architecture == 'x86_64' || github.repository_owner == 'python') + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + CC=clang-19 ./configure ${{ matrix.debug && '--with-pydebug' || '' }} + make all --jobs 4 + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index a08b32fa45db3e..b77046c320222f 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -80,6 +80,7 @@ def clean_lines(text): Python/deepfreeze/*.c Python/frozen_modules/*.h Python/generated_cases.c.h +Python/generated_tail_call_handlers.c.h Python/executor_cases.c.h Python/optimizer_cases.c.h diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 95c90bda70f352..6603d7a412980b 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -49,6 +49,13 @@ goto LABEL ## _tier_two; \ } while (0) +#undef CEVAL_GOTO +#define CEVAL_GOTO(LABEL) \ + do { \ + goto LABEL ## _tier_two; \ + } while (0) + + #undef GOTO_TIER_TWO #define GOTO_TIER_TWO(EXECUTOR) \ do { \ From e0d9e6c0524c1c3d6749d512509c64d72093d6c0 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 11 Jan 2025 03:32:01 +0800 Subject: [PATCH 036/110] Update tail-call.yml --- .github/workflows/tail-call.yml | 40 ++++++++++++++++----------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index 6920fd3e7348a3..4e28c7c3bb709f 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -1,4 +1,4 @@ -name: JIT +name: Tail Calling Interpreter on: pull_request: paths: @@ -40,28 +40,28 @@ jobs: llvm: - 19 include: - - target: i686-pc-windows-msvc/msvc - architecture: Win32 - runner: windows-latest - - target: x86_64-pc-windows-msvc/msvc - architecture: x64 - runner: windows-latest - - target: aarch64-pc-windows-msvc/msvc - architecture: ARM64 - runner: windows-latest - - target: x86_64-apple-darwin/clang - architecture: x86_64 - runner: macos-13 - - target: aarch64-apple-darwin/clang - architecture: aarch64 - runner: macos-14 +# - target: i686-pc-windows-msvc/msvc +# architecture: Win32 +# runner: windows-latest +# - target: x86_64-pc-windows-msvc/msvc +# architecture: x64 +# runner: windows-latest +# - target: aarch64-pc-windows-msvc/msvc +# architecture: ARM64 +# runner: windows-latest +# - target: x86_64-apple-darwin/clang +# architecture: x86_64 +# runner: macos-13 +# - target: aarch64-apple-darwin/clang +# architecture: aarch64 +# runner: macos-14 - target: x86_64-unknown-linux-gnu/gcc architecture: x86_64 runner: ubuntu-24.04 - - target: aarch64-unknown-linux-gnu/gcc - architecture: aarch64 - # Forks don't have access to our paid AArch64 runners. These jobs are skipped below: - runner: ${{ github.repository_owner == 'python' && 'ubuntu-24.04-aarch64' || 'ubuntu-24.04' }} +# - target: aarch64-unknown-linux-gnu/gcc +# architecture: aarch64 +# # Forks don't have access to our paid AArch64 runners. These jobs are skipped below: +# runner: ${{ github.repository_owner == 'python' && 'ubuntu-24.04-aarch64' || 'ubuntu-24.04' }} steps: - uses: actions/checkout@v4 with: From 5aec067fd67ae827c44534353161ff2dcce449f4 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 11 Jan 2025 03:33:03 +0800 Subject: [PATCH 037/110] fix name --- .github/workflows/tail-call.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index 4e28c7c3bb709f..5f348334d69f4a 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -1,4 +1,4 @@ -name: Tail Calling Interpreter +name: Tail calling interpreter on: pull_request: paths: From 6f7934b9dcea6b0ae15767a563684b94416cf080 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 11 Jan 2025 03:39:34 +0800 Subject: [PATCH 038/110] Simplify --- Python/ceval.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index adfb04ef69e442..7502b6fca823ab 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -786,11 +786,8 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) static inline PyObject *_TAIL_CALL_shim(TAIL_CALL_PARAMS) { opcode = next_instr->op.code; -#ifdef LLTRACE - return (INSTRUCTION_TABLE[opcode])(frame, stack_pointer, tstate, next_instr, opcode, next_instr->op.arg, lltrace); -#else - return (INSTRUCTION_TABLE[opcode])(frame, stack_pointer, tstate, next_instr, opcode, next_instr->op.arg); -#endif + oparg = next_instr->op.arg; + return (INSTRUCTION_TABLE[opcode])(TAIL_CALL_ARGS); } #endif From 142e56e84bf5f2c1c2270b70afebe76423e4f13a Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 11 Jan 2025 03:45:00 +0800 Subject: [PATCH 039/110] Address review --- .../2025-01-10-18-56-20.gh-issue-128563.baDvls.rst | 2 +- Python/ceval.c | 2 +- Python/ceval_macros.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst index 3e942358016411..9e5709013cecf0 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst @@ -1 +1 @@ -A new type of interpreter has been added to CPython. This interpreter uses tail calls for its instruction handlers. Preliminary benchmark results suggest 7.4-14.7% geometric mean faster on pyperformance (depending on platform), and up to 45% faster on Python-intensive workloads. Patch by Ken Jin, with ideas by Garret Gu, Haoran Xu, and Josh Haberman. +A new type of interpreter has been added to CPython. This interpreter uses tail calls for its instruction handlers. Preliminary benchmark results suggest 7-14% geometric mean faster on pyperformance (depending on platform), and up to 45% faster on Python-intensive workloads. Patch by Ken Jin, with ideas by Garret Gu, Haoran Xu, and Josh Haberman. diff --git a/Python/ceval.c b/Python/ceval.c index 7502b6fca823ab..9a64b48e2a53b4 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -797,7 +797,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _Py_EnsureTstateNotNULL(tstate); CALL_STAT_INC(pyeval_calls); -#if USE_COMPUTED_GOTOS && !defined(Py_TAIL_CALL_INTERP) +#if USE_COMPUTED_GOTOS /* Import the static jump table */ #include "opcode_targets.h" #endif diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 1d8d8c527c43f8..075200554cc4ed 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -47,7 +47,7 @@ #define OR_DTRACE_LINE #endif -#ifdef HAVE_COMPUTED_GOTOS +#if defined(HAVE_COMPUTED_GOTOS) && !defined(Py_TAIL_CALL_INTERP) #ifndef USE_COMPUTED_GOTOS #define USE_COMPUTED_GOTOS 1 #endif From 5bc7d143fd542ad3932441ca62da2d27951282bf Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 11 Jan 2025 04:01:40 +0800 Subject: [PATCH 040/110] Address review again --- Tools/cases_generator/tier1_tail_call_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index 30e070c22acb42..ac71e84e8bde06 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -24,7 +24,7 @@ write_single_inst ) -DEFAULT_INPUT = ROOT / "Python/generated_tail_call_handlers.c.h" +DEFAULT_INPUT = ROOT / "Python/bytecodes.c" DEFAULT_OUTPUT = ROOT / "Python/generated_tail_call_handlers.c.h" DEFAULT_CEVAL_INPUT = ROOT / "Python/ceval.c" From 195ee879d90ef2b31179af4e332519dfd11020c2 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 11 Jan 2025 04:34:10 +0800 Subject: [PATCH 041/110] Address review --- .github/workflows/tail-call.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index 5f348334d69f4a..e309168576d7ca 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -74,7 +74,7 @@ jobs: if: runner.os == 'Windows' && matrix.architecture != 'ARM64' run: | choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 - ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} + ./PCbuild/build.bat --tail-calling-interp ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 # No tests (yet): @@ -82,7 +82,7 @@ jobs: if: runner.os == 'Windows' && matrix.architecture == 'ARM64' run: | choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 - ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} + ./PCbuild/build.bat --tail-calling-interp ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. # This is a bug in the macOS runner image where the pre-installed Python is installed in the same @@ -95,7 +95,7 @@ jobs: find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete brew install llvm@${{ matrix.llvm }} export SDKROOT="$(xcrun --show-sdk-path)" - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} + ./configure --with-tail-calling-interp ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 @@ -105,7 +105,7 @@ jobs: run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - CC=clang-19 ./configure ${{ matrix.debug && '--with-pydebug' || '' }} + CC=clang-19 ./configure --with-tail-calling-interp ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 From 9df3e753025e07e9b87314f07ef4093aaffba254 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 11 Jan 2025 08:07:47 +0800 Subject: [PATCH 042/110] Update 2025-01-10-18-56-20.gh-issue-128563.baDvls.rst --- .../2025-01-10-18-56-20.gh-issue-128563.baDvls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst index 9e5709013cecf0..0c73ca294b2cb2 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst @@ -1 +1 @@ -A new type of interpreter has been added to CPython. This interpreter uses tail calls for its instruction handlers. Preliminary benchmark results suggest 7-14% geometric mean faster on pyperformance (depending on platform), and up to 45% faster on Python-intensive workloads. Patch by Ken Jin, with ideas by Garret Gu, Haoran Xu, and Josh Haberman. +A new type of interpreter has been added to CPython. This interpreter uses tail calls for its instruction handlers. Preliminary benchmark results suggest 7-14% geometric mean faster on pyperformance (depending on platform), and up to 45% faster on Python-intensive workloads. This interpreter currently only works on newer compilers, such as ``clang-19``. Other compilers will continue using the old interpreter. Patch by Ken Jin, with ideas by Garret Gu, Haoran Xu, and Josh Haberman. From 2b4851e2422b0e83dddfcc0a0d569d311e49e739 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 12 Jan 2025 09:06:22 +0800 Subject: [PATCH 043/110] fix configure auto detection --- configure | 23 ++++++++++++++--------- configure.ac | 21 ++++++++++++++------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/configure b/configure index 5808e6298efcfd..c0702cf05ec624 100755 --- a/configure +++ b/configure @@ -1122,7 +1122,7 @@ with_platlibdir with_wheel_pkg_dir with_readline with_computed_gotos -with_tail_calling_interp +with_tail_call_interp with_ensurepip with_openssl with_openssl_rpath @@ -1930,7 +1930,7 @@ Optional Packages: use libedit for backend or disable readline module --with-computed-gotos enable computed gotos in evaluation loop (enabled by default on supported compilers) - --tail-calling-interp enable tail-calling interpreter in evaluation loop + --tail-call-interp enable tail-calling interpreter in evaluation loop and rest of CPython (enabled by default on supported compilers) --with-ensurepip[=install|upgrade|no] @@ -29237,14 +29237,14 @@ printf "%s\n" "#define HAVE_COMPUTED_GOTOS 1" >>confdefs.h esac -# Check for --with-tail-calling-interp -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-tail-calling-interp" >&5 -printf %s "checking for --with-tail-calling-interp... " >&6; } +# Check for --with-tail-call-interp +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-tail-call-interp" >&5 +printf %s "checking for --with-tail-call-interp... " >&6; } -# Check whether --with-tail-calling-interp was given. -if test ${with_tail_calling_interp+y} +# Check whether --with-tail-call-interp was given. +if test ${with_tail_call_interp+y} then : - withval=$with_tail_calling_interp; + withval=$with_tail_call_interp; if test "$withval" = yes then @@ -29307,14 +29307,19 @@ esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_tail_call" >&5 printf "%s\n" "$ac_cv_tail_call" >&6; } -if $tier2_flags == ""; then +# Do not enable tail-calling interpreter if tier 2 is enabled. +if ${tier2_flags:+false} : +then : + case "$ac_cv_tail_call" in yes*) printf "%s\n" "#define Py_TAIL_CALL_INTERP 1" >>confdefs.h esac + fi + case $ac_sys_system in AIX*) diff --git a/configure.ac b/configure.ac index 4de8d055a55c8a..ca16289f24c01b 100644 --- a/configure.ac +++ b/configure.ac @@ -7004,12 +7004,12 @@ case "$ac_cv_computed_gotos" in yes*) [Define if the C compiler supports computed gotos.]) esac -# Check for --with-tail-calling-interp -AC_MSG_CHECKING([for --with-tail-calling-interp]) +# Check for --with-tail-call-interp +AC_MSG_CHECKING([for --with-tail-call-interp]) AC_ARG_WITH( - [tail-calling-interp], + [tail-call-interp], [AS_HELP_STRING( - [--tail-calling-interp], + [--tail-call-interp], [enable tail-calling interpreter in evaluation loop and rest of CPython (enabled by default on supported compilers)] )], [ @@ -7053,12 +7053,19 @@ AC_COMPILE_IFELSE([AC_LANG_SOURCE([[[ else ac_cv_tail_call=no fi])) -if $tier2_flags == ""; then +# Do not enable tail-calling interpreter if tier 2 is enabled. +AS_VAR_IF( + [tier2_flags], + [], + [ case "$ac_cv_tail_call" in yes*) AC_DEFINE([Py_TAIL_CALL_INTERP], [1], [Define if the C compiler supports efficient proper tail calls.]) - esac -fi + esac + ], + [] +) + case $ac_sys_system in AIX*) From 1ae8fc09ecd794c85bcdc06283abad15ab994208 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 12 Jan 2025 09:07:34 +0800 Subject: [PATCH 044/110] fix workflow --- .github/workflows/tail-call.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index e309168576d7ca..dc070bd73edc4a 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -74,7 +74,7 @@ jobs: if: runner.os == 'Windows' && matrix.architecture != 'ARM64' run: | choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 - ./PCbuild/build.bat --tail-calling-interp ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} + ./PCbuild/build.bat --tail-call-interp ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 # No tests (yet): @@ -82,7 +82,7 @@ jobs: if: runner.os == 'Windows' && matrix.architecture == 'ARM64' run: | choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 - ./PCbuild/build.bat --tail-calling-interp ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} + ./PCbuild/build.bat --tail-call-interp ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. # This is a bug in the macOS runner image where the pre-installed Python is installed in the same @@ -95,7 +95,7 @@ jobs: find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete brew install llvm@${{ matrix.llvm }} export SDKROOT="$(xcrun --show-sdk-path)" - ./configure --with-tail-calling-interp ${{ matrix.debug && '--with-pydebug' || '' }} + ./configure --with-tail-call-interp ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 @@ -105,7 +105,7 @@ jobs: run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - CC=clang-19 ./configure --with-tail-calling-interp ${{ matrix.debug && '--with-pydebug' || '' }} + CC=clang-19 ./configure --with-tail-call-interp ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 From c58f2859554722f2e05b97396e61b863e1a748a8 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 12 Jan 2025 09:41:00 +0800 Subject: [PATCH 045/110] Test on macOS as well --- .github/workflows/tail-call.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index dc070bd73edc4a..882d203c85a548 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -30,10 +30,10 @@ jobs: # - i686-pc-windows-msvc/msvc # - x86_64-pc-windows-msvc/msvc # - aarch64-pc-windows-msvc/msvc -# - x86_64-apple-darwin/clang -# - aarch64-apple-darwin/clang + - x86_64-apple-darwin/clang + - aarch64-apple-darwin/clang - x86_64-unknown-linux-gnu/gcc -# - aarch64-unknown-linux-gnu/gcc + - aarch64-unknown-linux-gnu/gcc debug: - true # - false @@ -49,19 +49,19 @@ jobs: # - target: aarch64-pc-windows-msvc/msvc # architecture: ARM64 # runner: windows-latest -# - target: x86_64-apple-darwin/clang -# architecture: x86_64 -# runner: macos-13 -# - target: aarch64-apple-darwin/clang -# architecture: aarch64 -# runner: macos-14 + - target: x86_64-apple-darwin/clang + architecture: x86_64 + runner: macos-13 + - target: aarch64-apple-darwin/clang + architecture: aarch64 + runner: macos-14 - target: x86_64-unknown-linux-gnu/gcc architecture: x86_64 runner: ubuntu-24.04 -# - target: aarch64-unknown-linux-gnu/gcc -# architecture: aarch64 -# # Forks don't have access to our paid AArch64 runners. These jobs are skipped below: -# runner: ${{ github.repository_owner == 'python' && 'ubuntu-24.04-aarch64' || 'ubuntu-24.04' }} + - target: aarch64-unknown-linux-gnu/gcc + architecture: aarch64 + # Forks don't have access to our paid AArch64 runners. These jobs are skipped below: + runner: ${{ github.repository_owner == 'python' && 'ubuntu-24.04-aarch64' || 'ubuntu-24.04' }} steps: - uses: actions/checkout@v4 with: From dfa2af74ab778b003ebcd52b18654bc07131699b Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 12 Jan 2025 09:49:26 +0800 Subject: [PATCH 046/110] Specify clang-19 --- .github/workflows/tail-call.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index 882d203c85a548..fa96b3262d5469 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -95,7 +95,7 @@ jobs: find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete brew install llvm@${{ matrix.llvm }} export SDKROOT="$(xcrun --show-sdk-path)" - ./configure --with-tail-call-interp ${{ matrix.debug && '--with-pydebug' || '' }} + CC=clang-19 ./configure --with-tail-call-interp ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 From 1fc4df54f3e344a1b4f6be35d1d6975a0f903fe8 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 12 Jan 2025 09:54:35 +0800 Subject: [PATCH 047/110] Try fix Apple --- .github/workflows/tail-call.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index fa96b3262d5469..23d5ea9b6ad73c 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -95,6 +95,7 @@ jobs: find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete brew install llvm@${{ matrix.llvm }} export SDKROOT="$(xcrun --show-sdk-path)" + echo 'export PATH="/opt/homebrew/opt/llvm/bin:$PATH"' CC=clang-19 ./configure --with-tail-call-interp ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 From dec29edee01269045e84ed3deda091e9496be020 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 12 Jan 2025 09:57:29 +0800 Subject: [PATCH 048/110] add to PATH --- .github/workflows/tail-call.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index 23d5ea9b6ad73c..259b941fdbdd7a 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -95,7 +95,7 @@ jobs: find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete brew install llvm@${{ matrix.llvm }} export SDKROOT="$(xcrun --show-sdk-path)" - echo 'export PATH="/opt/homebrew/opt/llvm/bin:$PATH"' + export PATH="/opt/homebrew/opt/llvm/bin:$PATH" CC=clang-19 ./configure --with-tail-call-interp ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 From 8d298954ca2e7857e95e95e005cb33eb5dbf98e5 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 12 Jan 2025 10:03:20 +0800 Subject: [PATCH 049/110] Fix on x86_64 Apple --- .github/workflows/tail-call.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index 259b941fdbdd7a..b0407c38dc3301 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -96,6 +96,7 @@ jobs: brew install llvm@${{ matrix.llvm }} export SDKROOT="$(xcrun --show-sdk-path)" export PATH="/opt/homebrew/opt/llvm/bin:$PATH" + export PATH="/usr/local/opt/llvm/bin:$PATH CC=clang-19 ./configure --with-tail-call-interp ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 From b1ae5707e7d51841887ddb366b3a9785527afd64 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 12 Jan 2025 10:06:10 +0800 Subject: [PATCH 050/110] missing " --- .github/workflows/tail-call.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index b0407c38dc3301..4fc7bcfa4bc15c 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -96,7 +96,7 @@ jobs: brew install llvm@${{ matrix.llvm }} export SDKROOT="$(xcrun --show-sdk-path)" export PATH="/opt/homebrew/opt/llvm/bin:$PATH" - export PATH="/usr/local/opt/llvm/bin:$PATH + export PATH="/usr/local/opt/llvm/bin:$PATH" CC=clang-19 ./configure --with-tail-call-interp ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 From 7a95b7e3d4d6799137cd5b452a30166b8795d437 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 12 Jan 2025 10:22:07 +0800 Subject: [PATCH 051/110] allow computed gotos with tail calls --- Python/ceval.c | 2 +- Python/ceval_macros.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 9a64b48e2a53b4..7502b6fca823ab 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -797,7 +797,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _Py_EnsureTstateNotNULL(tstate); CALL_STAT_INC(pyeval_calls); -#if USE_COMPUTED_GOTOS +#if USE_COMPUTED_GOTOS && !defined(Py_TAIL_CALL_INTERP) /* Import the static jump table */ #include "opcode_targets.h" #endif diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 075200554cc4ed..641c239602fea5 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -47,7 +47,7 @@ #define OR_DTRACE_LINE #endif -#if defined(HAVE_COMPUTED_GOTOS) && !defined(Py_TAIL_CALL_INTERP) +#ifdef HAVE_COMPUTED_GOTO #ifndef USE_COMPUTED_GOTOS #define USE_COMPUTED_GOTOS 1 #endif From d26ef11ad0259e450b53be33b87011635d4b3dce Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 12 Jan 2025 10:51:49 +0800 Subject: [PATCH 052/110] add ceval.c to list of paths --- .github/workflows/tail-call.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index 4fc7bcfa4bc15c..2dfd1beeee3f9a 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -3,10 +3,12 @@ on: pull_request: paths: - 'Python/bytecodes.c' + - 'Python/ceval.c' - 'Python/generated_tail_call_handlers.c.h' push: paths: - 'Python/bytecodes.c' + - 'Python/ceval.c' - 'Python/generated_tail_call_handlers.c.h' workflow_dispatch: From eac485f92703e3ac1ecddd2aa8e24da9881b48a5 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 14 Jan 2025 01:57:37 +0800 Subject: [PATCH 053/110] Fix upstream, fix warnings --- Python/generated_tail_call_handlers.c.h | 103 +++++++++++------- .../tier1_tail_call_generator.py | 6 +- 2 files changed, 69 insertions(+), 40 deletions(-) diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index b3e71af5c701b1..f6cd34b519f537 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -319,8 +319,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_ assert(PyUnicode_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = PyUnicode_Concat(left_o, right_o); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); if (res_o == NULL) CEVAL_GOTO(pop_2_error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -375,7 +375,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TA * that the string is safe to mutate. */ assert(Py_REFCNT(left_o) >= 2); - PyStackRef_CLOSE(left); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); PyUnicode_Append(&temp, right_o); *target_local = PyStackRef_FromPyObjectSteal(temp); @@ -1122,10 +1122,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS){ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } + PyStackRef_CLOSE(callable[0]); { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1161,10 +1161,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS){ } } assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1511,19 +1511,20 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyType_Check(callable_o), CALL); + PyTypeObject *tp = (PyTypeObject *)callable_o; int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } - DEOPT_IF(!PyType_Check(callable_o), CALL); - PyTypeObject *tp = (PyTypeObject *)callable_o; DEOPT_IF(tp->tp_vectorcall == NULL, CALL); STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); + PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -1539,7 +1540,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); + PyStackRef_CLOSE(arguments[i]); } PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { @@ -1590,8 +1591,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARA /* Builtin METH_FASTCALL functions, without keywords */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); @@ -1599,10 +1601,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARA STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); + PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -1622,7 +1624,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARA assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); + PyStackRef_CLOSE(arguments[i]); } PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { @@ -1689,7 +1691,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS( STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); + PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -1830,6 +1832,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (PyTuple_CheckExact(callargs_o)) { tuple = callargs; + kwargs_out = kwargs_in; } else { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1844,10 +1847,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM if (tuple_o == NULL) { CEVAL_GOTO(error); } + kwargs_out = kwargs_in; PyStackRef_CLOSE(callargs); tuple = PyStackRef_FromPyObjectSteal(tuple_o); } - kwargs_out = kwargs_in; } // _DO_CALL_FUNCTION_EX { @@ -2032,16 +2035,17 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS /* isinstance(o, o2) */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } DEOPT_IF(total_args != 2, CALL); PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL); STAT_INC(CALL, hit); - _PyStackRef cls_stackref = args[1]; - _PyStackRef inst_stackref = args[0]; + _PyStackRef cls_stackref = arguments[1]; + _PyStackRef inst_stackref = arguments[0]; _PyFrame_SetStackPointer(frame, stack_pointer); int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2050,9 +2054,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS } res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(inst_stackref); - PyStackRef_CLOSE(cls_stackref); PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2157,7 +2163,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); + PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -2199,10 +2205,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ } PyStackRef_CLOSE(kwnames); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2358,7 +2364,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); + PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -2535,8 +2541,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ if (res_o == NULL) { CEVAL_GOTO(error); } - PyStackRef_CLOSE(callable[0]); PyStackRef_CLOSE(arg_stackref); + PyStackRef_CLOSE(callable[0]); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2616,7 +2622,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); + PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -2701,7 +2707,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); + PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -2934,7 +2940,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PA STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_CLOSE(self_or_null[0]); + PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } @@ -2952,10 +2958,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PA stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -4644,10 +4650,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARA /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } + PyStackRef_CLOSE(callable[0]); { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -4683,10 +4689,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARA } } assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } + PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -6077,8 +6083,18 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS){ next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); PREDICTED(LOAD_CONST); + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; _PyStackRef value; - value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); + /* We can't do this in the bytecode compiler as + * marshalling can intern strings and make them immortal. */ + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectNew(obj); + #if ENABLE_SPECIALIZATION + if (this_instr->op.code == LOAD_CONST) { + this_instr->op.code = _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL; + } + #endif stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -6100,6 +6116,20 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_PA DISPATCH(); } +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_MORTAL(TAIL_CALL_PARAMS){ + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST_MORTAL); + static_assert(0 == 0, "incorrect cache size"); + _PyStackRef value; + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectNew(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); +} + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 1; @@ -6680,11 +6710,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL PyObject *attr_o = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); if (attr_o == NULL) { - PyStackRef_CLOSE(self_st); - CEVAL_GOTO(pop_3_error); + CEVAL_GOTO(error); } if (method_found) { self_or_null = self_st; // transfer ownership @@ -6692,6 +6719,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL PyStackRef_CLOSE(self_st); self_or_null = PyStackRef_NULL; } + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(global_super_st); attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; @@ -8603,6 +8632,7 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [LOAD_COMMON_CONSTANT] = _TAIL_CALL_LOAD_COMMON_CONSTANT, [LOAD_CONST] = _TAIL_CALL_LOAD_CONST, [LOAD_CONST_IMMORTAL] = _TAIL_CALL_LOAD_CONST_IMMORTAL, + [LOAD_CONST_MORTAL] = _TAIL_CALL_LOAD_CONST_MORTAL, [LOAD_DEREF] = _TAIL_CALL_LOAD_DEREF, [LOAD_FAST] = _TAIL_CALL_LOAD_FAST, [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_LOAD_FAST_AND_CLEAR, @@ -8714,7 +8744,6 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [146] = _TAIL_CALL_UNKNOWN_OPCODE, [147] = _TAIL_CALL_UNKNOWN_OPCODE, [148] = _TAIL_CALL_UNKNOWN_OPCODE, - [228] = _TAIL_CALL_UNKNOWN_OPCODE, [229] = _TAIL_CALL_UNKNOWN_OPCODE, [230] = _TAIL_CALL_UNKNOWN_OPCODE, [231] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index ac71e84e8bde06..dca3e53f9dce04 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -37,8 +37,8 @@ def generate_label_handlers(infile: TextIO, outfile: TextIO) -> None: out = CWriter(outfile, 0, False) str_in = infile.read() # https://stackoverflow.com/questions/8303488/regex-to-match-any-character-including-new-lines - eval_framedefault = re.findall("_PyEval_EvalFrameDefault\(.*\)\n({[\s\S]*\/\* END_BASE_INTERPRETER \*\/)", str_in)[0] - function_protos = re.findall(f"{TARGET_LABEL}\((\w+)\):", eval_framedefault) + eval_framedefault = re.findall(r"_PyEval_EvalFrameDefault\(.*\)\n({[\s\S]*\/\* END_BASE_INTERPRETER \*\/)", str_in)[0] + function_protos = re.findall(rf"{TARGET_LABEL}\((\w+)\):", eval_framedefault) for proto in function_protos: out.emit(f"{function_proto(proto)};\n") out.emit("\n") @@ -53,7 +53,7 @@ def generate_label_handlers(infile: TextIO, outfile: TextIO) -> None: for line in lines: if TARGET_LABEL in line: break - if label := re.findall("goto (\w+);", line): + if label := re.findall(r"goto (\w+);", line): out.emit(f"CEVAL_GOTO({label[0]});\n") else: out.emit_text(line) From 910bd8835f816a5879bc2501647c6e35943ff346 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 14 Jan 2025 05:56:53 +0800 Subject: [PATCH 054/110] cleanup --- .github/workflows/tail-call.yml | 6 +++-- Python/ceval_macros.h | 40 ++++++++++++++++----------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index 2dfd1beeee3f9a..8bb290a0275ea8 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -3,12 +3,14 @@ on: pull_request: paths: - 'Python/bytecodes.c' - - 'Python/ceval.c' + - 'Python/ceval.c' + - 'Python/ceval_macros.h' - 'Python/generated_tail_call_handlers.c.h' push: paths: - 'Python/bytecodes.c' - 'Python/ceval.c' + - 'Python/ceval_macros.h' - 'Python/generated_tail_call_handlers.c.h' workflow_dispatch: @@ -38,7 +40,7 @@ jobs: - aarch64-unknown-linux-gnu/gcc debug: - true -# - false + - false llvm: - 19 include: diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 641c239602fea5..e537e47b1c05d6 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -150,13 +150,13 @@ do { \ #ifdef Py_TAIL_CALL_INTERP #ifdef LLTRACE -#define DISPATCH_INLINED(NEW_FRAME) \ - do { \ - assert(tstate->interp->eval_frame == NULL); \ +#define DISPATCH_INLINED(NEW_FRAME) \ + do { \ + assert(tstate->interp->eval_frame == NULL); \ _PyFrame_SetStackPointer(frame, stack_pointer); \ - assert((NEW_FRAME)->previous == frame); \ - frame = tstate->current_frame = (NEW_FRAME); \ - CALL_STAT_INC(inlined_py_calls); \ + assert((NEW_FRAME)->previous == frame); \ + frame = tstate->current_frame = (NEW_FRAME); \ + CALL_STAT_INC(inlined_py_calls); \ if (_Py_EnterRecursivePy(tstate)) {\ CEVAL_GOTO(exit_unwind);\ } \ @@ -165,27 +165,25 @@ do { \ lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); \ if (lltrace < 0) { \ CEVAL_GOTO(exit_unwind); \ - } \ - NEXTOPARG(); \ - Py_MUSTTAIL \ - return (INSTRUCTION_TABLE[opcode])(TAIL_CALL_ARGS); \ + } \ + NEXTOPARG(); \ + DISPATCH_GOTO(); \ } while (0) #else -#define DISPATCH_INLINED(NEW_FRAME) \ -do { \ - assert(tstate->interp->eval_frame == NULL); \ +#define DISPATCH_INLINED(NEW_FRAME) \ +do { \ + assert(tstate->interp->eval_frame == NULL); \ _PyFrame_SetStackPointer(frame, stack_pointer); \ - assert((NEW_FRAME)->previous == frame); \ - frame = tstate->current_frame = (NEW_FRAME); \ - CALL_STAT_INC(inlined_py_calls); \ - if (_Py_EnterRecursivePy(tstate)) {\ - CEVAL_GOTO(exit_unwind);\ + assert((NEW_FRAME)->previous == frame); \ + frame = tstate->current_frame = (NEW_FRAME); \ + CALL_STAT_INC(inlined_py_calls); \ + if (_Py_EnterRecursivePy(tstate)) { \ + CEVAL_GOTO(exit_unwind); \ } \ next_instr = frame->instr_ptr; \ stack_pointer = _PyFrame_GetStackPointer(frame); \ - NEXTOPARG(); \ - Py_MUSTTAIL \ - return (INSTRUCTION_TABLE[opcode])(TAIL_CALL_ARGS); \ + NEXTOPARG(); \ + DISPATCH_GOTO(); \ } while (0) #endif #else From c1a6652b5bf9f65b22930ab06f63d93805d8908b Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 14 Jan 2025 07:01:12 +0800 Subject: [PATCH 055/110] Partial GCC 15.0 support --- Python/bytecodes.c | 12 +- Python/ceval_macros.h | 19 +- Python/executor_cases.c.h | 2 +- Python/generated_cases.c.h | 387 +- Python/generated_tail_call_handlers.c.h | 17820 ++++++++++------ Tools/cases_generator/generators_common.py | 11 +- .../tier1_tail_call_generator.py | 27 +- 7 files changed, 11117 insertions(+), 7161 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index bea0e61c6c586d..abb99e4fc5447f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1056,7 +1056,7 @@ dummy_func( if (err) { assert(oparg == 0); monitor_reraise(tstate, frame, this_instr); - CEVAL_GOTO(exception_unwind); + goto exception_unwind; } ERROR_IF(true, error); } @@ -1326,7 +1326,7 @@ dummy_func( assert(exc && PyExceptionInstance_Check(exc)); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); - CEVAL_GOTO(exception_unwind); + goto exception_unwind; } tier1 inst(END_ASYNC_FOR, (awaitable_st, exc_st -- )) { @@ -1342,7 +1342,7 @@ dummy_func( _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); INPUTS_DEAD(); - CEVAL_GOTO(exception_unwind); + goto exception_unwind; } } @@ -1365,7 +1365,7 @@ dummy_func( INPUTS_DEAD(); none = PyStackRef_NULL; value = PyStackRef_NULL; - CEVAL_GOTO(exception_unwind); + goto exception_unwind; } } @@ -4046,7 +4046,7 @@ dummy_func( PyObject *res_o = PyLong_FromSsize_t(len_i); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { - CEVAL_GOTO(error); + goto error; } PyStackRef_CLOSE(arg_stackref); DEAD(args); @@ -4785,7 +4785,7 @@ dummy_func( tstate, frame, this_instr, prev_instr); if (original_opcode < 0) { next_instr = this_instr+1; - CEVAL_GOTO(error); + goto error; } next_instr = frame->instr_ptr; if (next_instr != this_instr) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index e537e47b1c05d6..bfea96a8ccdccc 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -79,7 +79,8 @@ #endif #ifdef Py_TAIL_CALL_INTERP -# define Py_MUSTTAIL __attribute__((musttail)) + // Note: [[clang::musttail]] works for GCC 15, but not __attribute__((musttail)) at the moment. +# define Py_MUSTTAIL [[clang::musttail]] # define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) Py_PRESERVE_NONE_CC typedef PyObject* (*py_tail_call_funcptr)(TAIL_CALL_PARAMS); @@ -87,7 +88,7 @@ Py_MUSTTAIL \ return (INSTRUCTION_TABLE[opcode])(TAIL_CALL_ARGS); \ } while (0) -# define CEVAL_GOTO(name) do { \ +# define TAIL_CALL(name) do { \ Py_MUSTTAIL \ return (_TAIL_CALL_##name)(TAIL_CALL_ARGS); \ } while (0) @@ -95,11 +96,9 @@ #elif USE_COMPUTED_GOTOS # define TARGET(op) TARGET_##op: # define DISPATCH_GOTO() goto *opcode_targets[opcode] -# define CEVAL_GOTO(name) goto name; #else # define TARGET(op) case op: TARGET_##op: # define DISPATCH_GOTO() goto dispatch_opcode -# define CEVAL_GOTO(name) goto name; #endif #define TAIL_CALL_TARGET(name) name @@ -118,7 +117,7 @@ do { \ lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); \ if (lltrace < 0) { \ - CEVAL_GOTO(exit_unwind); \ + goto exit_unwind; \ } \ } while (0) #else @@ -158,13 +157,13 @@ do { \ frame = tstate->current_frame = (NEW_FRAME); \ CALL_STAT_INC(inlined_py_calls); \ if (_Py_EnterRecursivePy(tstate)) {\ - CEVAL_GOTO(exit_unwind);\ + goto exit_unwind;\ } \ next_instr = frame->instr_ptr; \ stack_pointer = _PyFrame_GetStackPointer(frame); \ lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); \ if (lltrace < 0) { \ - CEVAL_GOTO(exit_unwind); \ + goto exit_unwind; \ } \ NEXTOPARG(); \ DISPATCH_GOTO(); \ @@ -178,7 +177,7 @@ do { \ frame = tstate->current_frame = (NEW_FRAME); \ CALL_STAT_INC(inlined_py_calls); \ if (_Py_EnterRecursivePy(tstate)) { \ - CEVAL_GOTO(exit_unwind); \ + goto exit_unwind; \ } \ next_instr = frame->instr_ptr; \ stack_pointer = _PyFrame_GetStackPointer(frame); \ @@ -447,7 +446,7 @@ do { \ stack_pointer = _PyFrame_GetStackPointer(frame); \ if (next_instr == NULL) { \ next_instr = (dest)+1; \ - CEVAL_GOTO(error); \ + goto error; \ } \ } \ } while (0); @@ -491,7 +490,7 @@ do { \ tstate->previous_executor = NULL; \ frame = tstate->current_frame; \ if (next_instr == NULL) { \ - CEVAL_GOTO(resume_with_error); \ + goto resume_with_error; \ } \ stack_pointer = _PyFrame_GetStackPointer(frame); \ DISPATCH(); \ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0a2d14556baff0..67ba03bdeb1baa 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4941,7 +4941,7 @@ PyObject *res_o = PyLong_FromSsize_t(len_i); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { - CEVAL_GOTO(error); + goto error; } PyStackRef_CLOSE(arg_stackref); PyStackRef_CLOSE(callable[0]); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 5b5444830451a6..01d0962908b175 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -49,7 +49,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(lhs); PyStackRef_CLOSE(rhs); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); + if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -87,7 +87,7 @@ ((PyFloatObject *)left_o)->ob_fval + ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); + if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -124,7 +124,7 @@ PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); + if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -161,7 +161,7 @@ PyObject *res_o = PyUnicode_Concat(left_o, right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); + if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -220,7 +220,7 @@ PyUnicode_Append(&temp, right_o); *target_local = PyStackRef_FromPyObjectSteal(temp); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (PyStackRef_IsNull(*target_local)) CEVAL_GOTO(pop_2_error); + if (PyStackRef_IsNull(*target_local)) goto pop_2_error; #if TIER_ONE // The STORE_FAST is already done. This is done here in tier one, // and during trace projection in tier two: @@ -262,7 +262,7 @@ ((PyFloatObject *)left_o)->ob_fval * ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); + if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -299,7 +299,7 @@ PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); + if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -337,7 +337,7 @@ ((PyFloatObject *)left_o)->ob_fval - ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); + if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -374,7 +374,7 @@ PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); + if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -424,7 +424,7 @@ assert(WITHIN_STACK_BOUNDS()); } PyStackRef_CLOSE(container); - if (res_o == NULL) CEVAL_GOTO(pop_3_error); + if (res_o == NULL) goto pop_3_error; res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-3] = res; @@ -471,7 +471,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); + if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -506,7 +506,7 @@ } PyStackRef_CLOSE(dict_st); PyStackRef_CLOSE(sub_st); - if (rc <= 0) CEVAL_GOTO(pop_2_error); + if (rc <= 0) goto pop_2_error; // not found or error res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -689,7 +689,7 @@ if (list_o == NULL) { stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } list = PyStackRef_FromPyObjectSteal(list_o); stack_pointer[-oparg] = list; @@ -713,7 +713,7 @@ { stack_pointer += -oparg*2; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -729,7 +729,7 @@ if (map_o == NULL) { stack_pointer += -oparg*2; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } map = PyStackRef_FromPyObjectSteal(map_o); stack_pointer[-oparg*2] = map; @@ -755,7 +755,7 @@ { stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } } int err = 0; @@ -772,7 +772,7 @@ { stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } } set = PyStackRef_FromPyObjectSteal(set_o); @@ -803,7 +803,7 @@ if (slice_o == NULL) { stack_pointer += -2 - ((oparg == 3) ? 1 : 0); assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } slice = PyStackRef_FromPyObjectSteal(slice_o); stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; @@ -827,7 +827,7 @@ { stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } } PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); @@ -838,7 +838,7 @@ if (str_o == NULL) { stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } str = PyStackRef_FromPyObjectSteal(str_o); stack_pointer[-oparg] = str; @@ -858,7 +858,7 @@ if (tup_o == NULL) { stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } tup = PyStackRef_FromPyObjectSteal(tup_o); stack_pointer[-oparg] = tup; @@ -954,7 +954,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - CEVAL_GOTO(error); + goto error; } frame->return_offset = 4 ; DISPATCH_INLINED(new_frame); @@ -969,7 +969,7 @@ { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1008,7 +1008,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1023,7 +1023,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); + if (err != 0) goto error; stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -1076,7 +1076,7 @@ PyObject *self_o = PyType_GenericAlloc(tp, 0); stack_pointer = _PyFrame_GetStackPointer(frame); if (self_o == NULL) { - CEVAL_GOTO(error); + goto error; } self[0] = PyStackRef_FromPyObjectSteal(self_o); _PyStackRef temp = callable[0]; @@ -1104,7 +1104,7 @@ assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { _PyEval_FrameClearAndPop(tstate, shim); - CEVAL_GOTO(error); + goto error; } init_frame = temp; frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; @@ -1303,7 +1303,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - CEVAL_GOTO(error); + goto error; } new_frame = temp; } @@ -1371,7 +1371,7 @@ { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1386,7 +1386,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1401,7 +1401,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); + if (err != 0) goto error; stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -1451,7 +1451,7 @@ { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1470,7 +1470,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1485,7 +1485,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); + if (err != 0) goto error; stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -1538,7 +1538,7 @@ { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1554,7 +1554,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1569,7 +1569,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); + if (err != 0) goto error; stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -1622,7 +1622,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1637,7 +1637,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); + if (err != 0) goto error; stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -1679,13 +1679,13 @@ int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - CEVAL_GOTO(error); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *tuple_o = PySequence_Tuple(callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (tuple_o == NULL) { - CEVAL_GOTO(error); + goto error; } kwargs_out = kwargs_in; PyStackRef_CLOSE(callargs); @@ -1718,7 +1718,7 @@ frame, this_instr, func, arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - CEVAL_GOTO(error); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); result_o = PyObject_Call(func, callargs, kwargs); @@ -1765,7 +1765,7 @@ stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { - CEVAL_GOTO(error); + goto error; } assert( 1 == 1); frame->return_offset = 1; @@ -1789,7 +1789,7 @@ if (result_o == NULL) { stack_pointer += -3 - (oparg & 1); assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } result = PyStackRef_FromPyObjectSteal(result_o); } @@ -1804,7 +1804,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); + if (err != 0) goto error; stack_pointer += 2 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); } @@ -1827,7 +1827,7 @@ PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (res_o == NULL) CEVAL_GOTO(pop_1_error); + if (res_o == NULL) goto pop_1_error; res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; DISPATCH(); @@ -1850,7 +1850,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value2_st); PyStackRef_CLOSE(value1_st); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); + if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -1890,7 +1890,7 @@ int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); stack_pointer = _PyFrame_GetStackPointer(frame); if (retval < 0) { - CEVAL_GOTO(error); + goto error; } res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -1993,7 +1993,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - CEVAL_GOTO(error); + goto error; } assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4 ; @@ -2011,7 +2011,7 @@ { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } } stack_pointer[-1] = kwnames; @@ -2052,7 +2052,7 @@ if (res_o == NULL) { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2136,7 +2136,7 @@ stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - CEVAL_GOTO(error); + goto error; } new_frame = temp; } @@ -2212,7 +2212,7 @@ { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } } PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); @@ -2233,7 +2233,7 @@ if (res_o == NULL) { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2248,7 +2248,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); + if (err != 0) goto error; stack_pointer += 2 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -2312,7 +2312,7 @@ stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - CEVAL_GOTO(error); + goto error; } new_frame = temp; } @@ -2374,12 +2374,12 @@ Py_ssize_t len_i = PyObject_Length(arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (len_i < 0) { - CEVAL_GOTO(error); + goto error; } PyObject *res_o = PyLong_FromSsize_t(len_i); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { - CEVAL_GOTO(error); + goto error; } PyStackRef_CLOSE(arg_stackref); PyStackRef_CLOSE(callable[0]); @@ -2416,7 +2416,7 @@ UNLOCK_OBJECT(self_o); PyStackRef_CLOSE(self); PyStackRef_CLOSE(callable); - if (err) CEVAL_GOTO(pop_3_error); + if (err) goto pop_3_error; #if TIER_ONE // Skip the following POP_TOP. This is done here in tier one, and // during trace projection in tier two: @@ -2469,7 +2469,7 @@ { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2487,7 +2487,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2502,7 +2502,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); + if (err != 0) goto error; stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -2554,7 +2554,7 @@ { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2572,7 +2572,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2587,7 +2587,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); + if (err != 0) goto error; stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -2644,7 +2644,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2659,7 +2659,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); + if (err != 0) goto error; stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -2719,7 +2719,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2734,7 +2734,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); + if (err != 0) goto error; stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -2787,7 +2787,7 @@ { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2805,7 +2805,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2820,7 +2820,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); + if (err != 0) goto error; stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -2959,7 +2959,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - CEVAL_GOTO(error); + goto error; } new_frame = temp; } @@ -3016,7 +3016,7 @@ PyObject *res_o = PyObject_Str(arg_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(arg); - if (res_o == NULL) CEVAL_GOTO(pop_3_error); + if (res_o == NULL) goto pop_3_error; res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC @@ -3030,7 +3030,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); + if (err != 0) goto error; stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); } @@ -3067,7 +3067,7 @@ PyObject *res_o = PySequence_Tuple(arg_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(arg); - if (res_o == NULL) CEVAL_GOTO(pop_3_error); + if (res_o == NULL) goto pop_3_error; res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC @@ -3081,7 +3081,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); + if (err != 0) goto error; stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); } @@ -3138,7 +3138,7 @@ if (err < 0) { PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); - CEVAL_GOTO(pop_2_error); + goto pop_2_error; } PyObject *match_o = NULL; PyObject *rest_o = NULL; @@ -3148,9 +3148,9 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); - if (res < 0) CEVAL_GOTO(pop_2_error); + if (res < 0) goto pop_2_error; assert((match_o == NULL) == (rest_o == NULL)); - if (match_o == NULL) CEVAL_GOTO(pop_2_error); + if (match_o == NULL) goto pop_2_error; if (!Py_IsNone(match_o)) { stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -3184,7 +3184,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { PyStackRef_CLOSE(right); - CEVAL_GOTO(pop_1_error); + goto pop_1_error; } _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyErr_GivenExceptionMatches(left_o, right_o); @@ -3230,7 +3230,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); none = PyStackRef_NULL; value = PyStackRef_NULL; - CEVAL_GOTO(exception_unwind); + goto exception_unwind; } stack_pointer[-3] = none; stack_pointer[-2] = value; @@ -3277,7 +3277,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); + if (res_o == NULL) goto pop_2_error; if (oparg & 16) { stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -3285,7 +3285,7 @@ int res_bool = PyObject_IsTrue(res_o); stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(res_o); - if (res_bool < 0) CEVAL_GOTO(error); + if (res_bool < 0) goto error; res = res_bool ? PyStackRef_True : PyStackRef_False; } else { @@ -3456,7 +3456,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); - if (res < 0) CEVAL_GOTO(pop_2_error); + if (res < 0) goto pop_2_error; b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } stack_pointer[-2] = b; @@ -3485,7 +3485,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); - if (res < 0) CEVAL_GOTO(pop_2_error); + if (res < 0) goto pop_2_error; b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -3514,7 +3514,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); - if (res < 0) CEVAL_GOTO(pop_2_error); + if (res < 0) goto pop_2_error; b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -3536,7 +3536,7 @@ PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (result_o == NULL) CEVAL_GOTO(pop_1_error); + if (result_o == NULL) goto pop_1_error; result = PyStackRef_FromPyObjectSteal(result_o); stack_pointer[-1] = result; DISPATCH(); @@ -3586,7 +3586,7 @@ int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); - if (err) CEVAL_GOTO(pop_1_error); + if (err) goto pop_1_error; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -3604,7 +3604,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + goto error; } Py_DECREF(oldobj); DISPATCH(); @@ -3622,7 +3622,7 @@ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + goto error; } SETLOCAL(oparg, PyStackRef_NULL); DISPATCH(); @@ -3638,14 +3638,14 @@ stack_pointer = _PyFrame_GetStackPointer(frame); // Can't use ERROR_IF here. if (err < 0) { - CEVAL_GOTO(error); + goto error; } if (err == 0) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + goto error; } DISPATCH(); } @@ -3662,7 +3662,7 @@ _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_DelItem(ns, name); @@ -3674,7 +3674,7 @@ NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + goto error; } DISPATCH(); } @@ -3694,7 +3694,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); - if (err) CEVAL_GOTO(pop_2_error); + if (err) goto pop_2_error; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -3721,7 +3721,7 @@ _PyEval_FormatKwargsError(tstate, callable_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(update); - CEVAL_GOTO(pop_1_error); + goto pop_1_error; } PyStackRef_CLOSE(update); stack_pointer += -1; @@ -3754,7 +3754,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(update); - CEVAL_GOTO(pop_1_error); + goto pop_1_error; } PyStackRef_CLOSE(update); stack_pointer += -1; @@ -3786,7 +3786,7 @@ _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(exception_unwind); + goto exception_unwind; } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -3873,7 +3873,7 @@ "__init__() should return None, not '%.200s'", Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + goto error; } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3906,7 +3906,7 @@ PyObject *res_o = PyObject_Format(value_o, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (res_o == NULL) CEVAL_GOTO(pop_1_error); + if (res_o == NULL) goto pop_1_error; res = PyStackRef_FromPyObjectSteal(res_o); } else { @@ -3930,7 +3930,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); PyStackRef_CLOSE(fmt_spec); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); + if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -3977,7 +3977,7 @@ int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); stack_pointer = _PyFrame_GetStackPointer(frame); if (!matches) { - CEVAL_GOTO(error); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_MonitorRaise(tstate, frame, this_instr); @@ -4131,7 +4131,7 @@ r->start = value + r->step; r->len--; PyObject *res = PyLong_FromLong(value); - if (res == NULL) CEVAL_GOTO(error); + if (res == NULL) goto error; next = PyStackRef_FromPyObjectSteal(res); } stack_pointer[0] = next; @@ -4208,13 +4208,13 @@ type->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); - CEVAL_GOTO(pop_1_error); + goto pop_1_error; } _PyFrame_SetStackPointer(frame, stack_pointer); iter_o = (*getter)(obj_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); - if (iter_o == NULL) CEVAL_GOTO(pop_1_error); + if (iter_o == NULL) goto pop_1_error; if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { stack_pointer += -1; @@ -4226,7 +4226,7 @@ Py_TYPE(iter_o)->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(iter_o); - CEVAL_GOTO(error); + goto error; } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; @@ -4244,7 +4244,7 @@ PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); stack_pointer = _PyFrame_GetStackPointer(frame); if (awaitable_o == NULL) { - CEVAL_GOTO(error); + goto error; } awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); stack_pointer[0] = awaitable; @@ -4264,7 +4264,7 @@ PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); - if (iter_o == NULL) CEVAL_GOTO(pop_1_error); + if (iter_o == NULL) goto pop_1_error; iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; DISPATCH(); @@ -4282,7 +4282,7 @@ PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); - if (iter_o == NULL) CEVAL_GOTO(pop_1_error); + if (iter_o == NULL) goto pop_1_error; iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; DISPATCH(); @@ -4299,9 +4299,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) CEVAL_GOTO(error); + if (len_i < 0) goto error; PyObject *len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) CEVAL_GOTO(error); + if (len_o == NULL) goto error; len = PyStackRef_FromPyObjectSteal(len_o); stack_pointer[0] = len; stack_pointer += 1; @@ -4328,7 +4328,7 @@ "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + goto error; } iter = iterable; } @@ -4342,7 +4342,7 @@ PyObject *iter_o = PyObject_GetIter(iterable_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { - CEVAL_GOTO(error); + goto error; } iter = PyStackRef_FromPyObjectSteal(iter_o); PyStackRef_CLOSE(iterable); @@ -4363,7 +4363,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) CEVAL_GOTO(error); + if (res_o == NULL) goto error; res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; @@ -4388,7 +4388,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(level); PyStackRef_CLOSE(fromlist); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); + if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -4450,7 +4450,7 @@ frame, this_instr, function, arg0 ); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) CEVAL_GOTO(error); + if (err) goto error; } // _DO_CALL { @@ -4482,7 +4482,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - CEVAL_GOTO(error); + goto error; } frame->return_offset = 4 ; DISPATCH_INLINED(new_frame); @@ -4497,7 +4497,7 @@ { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4536,7 +4536,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4551,7 +4551,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); + if (err != 0) goto error; stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -4588,7 +4588,7 @@ tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) CEVAL_GOTO(error); + if (err) goto error; PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); GO_TO_INSTRUCTION(CALL_KW); } @@ -4609,7 +4609,7 @@ int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - CEVAL_GOTO(error); + goto error; } } PyStackRef_CLOSE(value); @@ -4634,7 +4634,7 @@ int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - CEVAL_GOTO(error); + goto error; } } val = value; @@ -4666,7 +4666,7 @@ int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); stack_pointer = _PyFrame_GetStackPointer(frame); if (!matches) { - CEVAL_GOTO(error); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_MonitorRaise(tstate, frame, this_instr); @@ -4691,7 +4691,7 @@ int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_opcode < 0) CEVAL_GOTO(error); + if (next_opcode < 0) goto error; next_instr = this_instr; if (_PyOpcode_Caches[next_opcode]) { PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); @@ -4715,7 +4715,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); + if (err != 0) goto error; } } // _MONITOR_JUMP_BACKWARD @@ -4754,7 +4754,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (original_opcode < 0) { next_instr = this_instr+1; - CEVAL_GOTO(error); + goto error; } next_instr = frame->instr_ptr; if (next_instr != this_instr) { @@ -4889,7 +4889,7 @@ _Py_CODEUNIT *bytecode = _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) CEVAL_GOTO(error); + if (bytecode == NULL) goto error; _PyFrame_SetStackPointer(frame, stack_pointer); ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4912,7 +4912,7 @@ int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - CEVAL_GOTO(error); + goto error; } next_instr = this_instr; DISPATCH(); @@ -4928,7 +4928,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); + if (err != 0) goto error; } } } @@ -4938,7 +4938,7 @@ int err = _Py_call_instrumentation( tstate, oparg > 0, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) CEVAL_GOTO(error); + if (err) goto error; if (frame->instr_ptr != this_instr) { /* Instrumentation has jumped */ next_instr = frame->instr_ptr; @@ -4963,7 +4963,7 @@ tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) CEVAL_GOTO(error); + if (err) goto error; } // _RETURN_VALUE { @@ -5009,7 +5009,7 @@ frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - CEVAL_GOTO(error); + goto error; } if (frame->instr_ptr != this_instr) { next_instr = frame->instr_ptr; @@ -5112,7 +5112,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); + if (err != 0) goto error; } } // _JUMP_BACKWARD @@ -5137,7 +5137,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (optimized <= 0) { this_instr[1].counter = restart_backoff_counter(counter); - if (optimized < 0) CEVAL_GOTO(error); + if (optimized < 0) goto error; } else { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5188,7 +5188,7 @@ list = stack_pointer[-2 - (oparg-1)]; int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), PyStackRef_AsPyObjectSteal(v)); - if (err < 0) CEVAL_GOTO(pop_1_error); + if (err < 0) goto pop_1_error; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -5222,7 +5222,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(iterable_st); - CEVAL_GOTO(pop_1_error); + goto pop_1_error; } assert(Py_IsNone(none_val)); PyStackRef_CLOSE(iterable_st); @@ -5286,7 +5286,7 @@ meth | NULL | arg1 | ... | argN */ PyStackRef_CLOSE(owner); - if (attr_o == NULL) CEVAL_GOTO(pop_1_error); + if (attr_o == NULL) goto pop_1_error; self_or_null = PyStackRef_NULL; } } @@ -5296,7 +5296,7 @@ attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); - if (attr_o == NULL) CEVAL_GOTO(pop_1_error); + if (attr_o == NULL) goto pop_1_error; /* We need to define self_or_null on all paths */ self_or_null = PyStackRef_NULL; } @@ -5881,13 +5881,13 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) CEVAL_GOTO(error); + if (err < 0) goto error; if (bc_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + goto error; } bc = PyStackRef_FromPyObjectSteal(bc_o); stack_pointer[0] = bc; @@ -5981,7 +5981,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + goto error; } value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; @@ -6030,7 +6030,7 @@ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + goto error; } value = PyStackRef_DUP(value_s); stack_pointer[0] = value; @@ -6073,7 +6073,7 @@ int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - CEVAL_GOTO(error); + goto error; } if (!value_o) { PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); @@ -6082,7 +6082,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + goto error; } } PyStackRef_CLOSE(class_dict_st); @@ -6104,7 +6104,7 @@ int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(mod_or_class_dict); - if (err < 0) CEVAL_GOTO(pop_1_error); + if (err < 0) goto pop_1_error; if (v_o == NULL) { if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) @@ -6125,7 +6125,7 @@ NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); } - CEVAL_GOTO(error); + goto error; } } else { @@ -6136,20 +6136,20 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) CEVAL_GOTO(error); + if (err < 0) goto error; if (v_o == NULL) { /* namespace 2: builtins */ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) CEVAL_GOTO(error); + if (err < 0) goto error; if (v_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + goto error; } } } @@ -6197,7 +6197,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*res)) CEVAL_GOTO(error); + if (PyStackRef_IsNull(*res)) goto error; null = PyStackRef_NULL; } if (oparg & 1) stack_pointer[1] = null; @@ -6311,7 +6311,7 @@ _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + goto error; } locals = PyStackRef_FromPyObjectNew(l); stack_pointer[0] = locals; @@ -6329,7 +6329,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *v_o = _PyEval_LoadName(tstate, frame, name); stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) CEVAL_GOTO(error); + if (v_o == NULL) goto error; v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; @@ -6376,7 +6376,7 @@ Py_TYPE(owner_o)->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); } - CEVAL_GOTO(error); + goto error; } attr = PyStackRef_FromPyObjectSteal(attr_o); self_or_null = self_or_null_o == NULL ? @@ -6436,7 +6436,7 @@ PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); - CEVAL_GOTO(pop_3_error); + goto pop_3_error; } } // we make no attempt to optimize here; specializations should @@ -6468,7 +6468,7 @@ PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); - if (super == NULL) CEVAL_GOTO(pop_3_error); + if (super == NULL) goto pop_3_error; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); @@ -6476,7 +6476,7 @@ PyObject *attr_o = PyObject_GetAttr(super, name); stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(super); - if (attr_o == NULL) CEVAL_GOTO(error); + if (attr_o == NULL) goto error; attr = PyStackRef_FromPyObjectSteal(attr_o); null = PyStackRef_NULL; } @@ -6514,7 +6514,7 @@ PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); - if (attr == NULL) CEVAL_GOTO(pop_3_error); + if (attr == NULL) goto pop_3_error; attr_st = PyStackRef_FromPyObjectSteal(attr); stack_pointer[-3] = attr_st; stack_pointer += -2; @@ -6551,7 +6551,7 @@ Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { - CEVAL_GOTO(error); + goto error; } if (method_found) { self_or_null = self_st; // transfer ownership @@ -6578,7 +6578,7 @@ PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *cell = PyCell_New(initial); if (cell == NULL) { - CEVAL_GOTO(error); + goto error; } SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell)); DISPATCH(); @@ -6597,7 +6597,7 @@ PyFunction_New(codeobj, GLOBALS()); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(codeobj_st); - if (func_obj == NULL) CEVAL_GOTO(pop_1_error); + if (func_obj == NULL) goto pop_1_error; _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); @@ -6626,7 +6626,7 @@ PyStackRef_AsPyObjectSteal(value) ); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(pop_2_error); + if (err != 0) goto pop_2_error; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -6660,7 +6660,7 @@ attrs = PyStackRef_FromPyObjectSteal(attrs_o); } else { - if (_PyErr_Occurred(tstate)) CEVAL_GOTO(pop_3_error); + if (_PyErr_Occurred(tstate)) goto pop_3_error; // Error! attrs = PyStackRef_None; // Failure! } @@ -6684,7 +6684,7 @@ PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (values_or_none_o == NULL) CEVAL_GOTO(error); + if (values_or_none_o == NULL) goto error; values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -6934,9 +6934,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(exception_unwind); + goto exception_unwind; } - CEVAL_GOTO(error); + goto error; } TARGET(RERAISE) { @@ -6967,7 +6967,7 @@ _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(exc); - CEVAL_GOTO(error); + goto error; } stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -6979,8 +6979,7 @@ _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(exception_unwind); - DISPATCH(); + goto exception_unwind; } TARGET(RESERVED) { @@ -7008,7 +7007,7 @@ _Py_CODEUNIT *bytecode = _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) CEVAL_GOTO(error); + if (bytecode == NULL) goto error; _PyFrame_SetStackPointer(frame, stack_pointer); ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7031,7 +7030,7 @@ int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - CEVAL_GOTO(error); + goto error; } next_instr = this_instr; DISPATCH(); @@ -7055,7 +7054,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); + if (err != 0) goto error; } } } @@ -7092,7 +7091,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); stack_pointer = _PyFrame_GetStackPointer(frame); - if (gen == NULL) CEVAL_GOTO(error); + if (gen == NULL) goto error; assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; @@ -7225,7 +7224,7 @@ } else { PyStackRef_CLOSE(v); - CEVAL_GOTO(pop_1_error); + goto pop_1_error; } } PyStackRef_CLOSE(v); @@ -7297,24 +7296,24 @@ _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + goto error; } /* check if __annotations__ in locals()... */ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) CEVAL_GOTO(error); + if (err < 0) goto error; if (ann_dict == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); ann_dict = PyDict_New(); stack_pointer = _PyFrame_GetStackPointer(frame); - if (ann_dict == NULL) CEVAL_GOTO(error); + if (ann_dict == NULL) goto error; _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(ann_dict); - if (err) CEVAL_GOTO(error); + if (err) goto error; } else { Py_DECREF(ann_dict); @@ -7335,7 +7334,7 @@ PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); - if (err) CEVAL_GOTO(pop_1_error); + if (err) goto pop_1_error; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -7378,7 +7377,7 @@ PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); - if (err < 0) CEVAL_GOTO(pop_1_error); + if (err < 0) goto pop_1_error; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -7422,7 +7421,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); PyStackRef_CLOSE(owner); - if (err) CEVAL_GOTO(pop_2_error); + if (err) goto pop_2_error; } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -7654,7 +7653,7 @@ int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); - if (err) CEVAL_GOTO(pop_1_error); + if (err) goto pop_1_error; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -7675,7 +7674,7 @@ "no locals found when storing %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); - CEVAL_GOTO(pop_1_error); + goto pop_1_error; } if (PyDict_CheckExact(ns)) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -7688,7 +7687,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(v); - if (err) CEVAL_GOTO(pop_1_error); + if (err) goto pop_1_error; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -7735,7 +7734,7 @@ } PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); - if (err) CEVAL_GOTO(pop_4_error); + if (err) goto pop_4_error; } stack_pointer += -4; assert(WITHIN_STACK_BOUNDS()); @@ -7780,7 +7779,7 @@ PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); - if (err) CEVAL_GOTO(pop_3_error); + if (err) goto pop_3_error; } stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); @@ -7808,7 +7807,7 @@ PyStackRef_AsPyObjectSteal(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(dict_st); - if (err) CEVAL_GOTO(pop_3_error); + if (err) goto pop_3_error; stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -7903,7 +7902,7 @@ int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (err < 0) CEVAL_GOTO(pop_1_error); + if (err < 0) goto pop_1_error; res = err ? PyStackRef_True : PyStackRef_False; } stack_pointer[-1] = res; @@ -8050,7 +8049,7 @@ PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (res_o == NULL) CEVAL_GOTO(pop_1_error); + if (res_o == NULL) goto pop_1_error; res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; DISPATCH(); @@ -8067,7 +8066,7 @@ PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (res_o == NULL) CEVAL_GOTO(pop_1_error); + if (res_o == NULL) goto pop_1_error; res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; DISPATCH(); @@ -8100,7 +8099,7 @@ int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); - if (res == 0) CEVAL_GOTO(pop_1_error); + if (res == 0) goto pop_1_error; stack_pointer += (oparg & 0xFF) + (oparg >> 8); assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -8142,7 +8141,7 @@ int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); - if (res == 0) CEVAL_GOTO(pop_1_error); + if (res == 0) goto pop_1_error; } stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -8269,7 +8268,7 @@ PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) CEVAL_GOTO(error); + if (res_o == NULL) goto error; res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index f6cd34b519f537..57eb2e7cda4fc7 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -22,25 +22,25 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_resume_with_error(TAIL_CALL_PARA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_4_error(TAIL_CALL_PARAMS) { STACK_SHRINK(1); - CEVAL_GOTO(pop_3_error); + TAIL_CALL(pop_3_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_3_error(TAIL_CALL_PARAMS) { STACK_SHRINK(1); - CEVAL_GOTO(pop_2_error); + TAIL_CALL(pop_2_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS) { STACK_SHRINK(1); - CEVAL_GOTO(pop_1_error); + TAIL_CALL(pop_1_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS) { STACK_SHRINK(1); - CEVAL_GOTO(error); + TAIL_CALL(error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS) @@ -64,7 +64,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS) } } _PyEval_MonitorRaise(tstate, frame, next_instr-1); - CEVAL_GOTO(exception_unwind); + TAIL_CALL(exception_unwind); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS) @@ -85,7 +85,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAM assert(STACK_LEVEL() == 0); _PyFrame_SetStackPointer(frame, stack_pointer); monitor_unwind(tstate, frame, next_instr-1); - CEVAL_GOTO(exit_unwind); + TAIL_CALL(exit_unwind); } assert(STACK_LEVEL() >= level); @@ -97,7 +97,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAM int frame_lasti = _PyInterpreterFrame_LASTI(frame); PyObject *lasti = PyLong_FromLong(frame_lasti); if (lasti == NULL) { - CEVAL_GOTO(exception_unwind); + TAIL_CALL(exception_unwind); } PUSH(PyStackRef_FromPyObjectSteal(lasti)); } @@ -111,7 +111,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAM next_instr = _PyFrame_GetBytecode(frame) + handler; if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - CEVAL_GOTO(exception_unwind); + TAIL_CALL(exception_unwind); } /* Resume normal execution */ #ifdef LLTRACE @@ -135,7 +135,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAM #endif } - CEVAL_GOTO(exit_unwind); + TAIL_CALL(exit_unwind); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS) @@ -155,14 +155,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS) return NULL; } - CEVAL_GOTO(resume_with_error); + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_resume_with_error(TAIL_CALL_PARAMS) { next_instr = frame->instr_ptr; stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + TAIL_CALL(error); /* END_BASE_INTERPRETER */ } @@ -170,1593 +170,3057 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_resume_with_error(TAIL_CALL_PARA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP); - PREDICTED(BINARY_OP); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef lhs; - _PyStackRef rhs; - _PyStackRef res; - // _SPECIALIZE_BINARY_OP - { - rhs = stack_pointer[-1]; - lhs = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP); + PREDICTED(BINARY_OP); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef lhs; + _PyStackRef rhs; + _PyStackRef res; + // _SPECIALIZE_BINARY_OP + { + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(BINARY_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + assert(NB_ADD <= oparg); + assert(oparg <= NB_INPLACE_XOR); + } + // _BINARY_OP + { + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); + assert(_PyEval_BinaryOps[oparg]); _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + PyStackRef_CLOSE(lhs); + PyStackRef_CLOSE(rhs); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); } - OPCODE_DEFERRED_INC(BINARY_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - assert(NB_ADD <= oparg); - assert(oparg <= NB_INPLACE_XOR); - } - // _BINARY_OP - { - PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); - PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); - assert(_PyEval_BinaryOps[oparg]); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(lhs); - PyStackRef_CLOSE(rhs); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_FLOAT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); - } - /* Skip 1 cache entry */ - // _BINARY_OP_ADD_FLOAT { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval + - ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - res = PyStackRef_FromPyObjectSteal(res_o); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_FLOAT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_ADD_FLOAT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_ADD_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_INT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); - } - /* Skip 1 cache entry */ - // _BINARY_OP_ADD_INT { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - res = PyStackRef_FromPyObjectSteal(res_o); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_INT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_ADD_INT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_UNICODE - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); - } - /* Skip 1 cache entry */ - // _BINARY_OP_ADD_UNICODE { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyUnicode_Concat(left_o, right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - res = PyStackRef_FromPyObjectSteal(res_o); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_UNICODE + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_ADD_UNICODE + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - // _GUARD_BOTH_UNICODE - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); - } - /* Skip 1 cache entry */ - // _BINARY_OP_INPLACE_ADD_UNICODE { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(right_o)); - int next_oparg; - #if TIER_ONE - assert(next_instr->op.code == STORE_FAST); - next_oparg = next_instr->op.arg; - #else - next_oparg = CURRENT_OPERAND0(); - #endif - _PyStackRef *target_local = &GETLOCAL(next_oparg); - DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP); - STAT_INC(BINARY_OP, hit); - /* Handle `left = left + right` or `left += right` for str. - * - * When possible, extend `left` in place rather than - * allocating a new PyUnicodeObject. This attempts to avoid - * quadratic behavior when one neglects to use str.join(). - * - * If `left` has only two references remaining (one from - * the stack, one in the locals), DECREFing `left` leaves - * only the locals reference, so PyUnicode_Append knows - * that the string is safe to mutate. - */ - assert(Py_REFCNT(left_o) >= 2); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); - PyUnicode_Append(&temp, right_o); - *target_local = PyStackRef_FromPyObjectSteal(temp); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (PyStackRef_IsNull(*target_local)) CEVAL_GOTO(pop_2_error); - #if TIER_ONE - // The STORE_FAST is already done. This is done here in tier one, - // and during trace projection in tier two: - assert(next_instr->op.code == STORE_FAST); - SKIP_OVER(1); - #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + // _GUARD_BOTH_UNICODE + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_INPLACE_ADD_UNICODE + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); + int next_oparg; + #if TIER_ONE + assert(next_instr->op.code == STORE_FAST); + next_oparg = next_instr->op.arg; + #else + next_oparg = CURRENT_OPERAND0(); + #endif + _PyStackRef *target_local = &GETLOCAL(next_oparg); + DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP); + STAT_INC(BINARY_OP, hit); + /* Handle `left = left + right` or `left += right` for str. + * + * When possible, extend `left` in place rather than + * allocating a new PyUnicodeObject. This attempts to avoid + * quadratic behavior when one neglects to use str.join(). + * + * If `left` has only two references remaining (one from + * the stack, one in the locals), DECREFing `left` leaves + * only the locals reference, so PyUnicode_Append knows + * that the string is safe to mutate. + */ + assert(Py_REFCNT(left_o) >= 2); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); + PyUnicode_Append(&temp, right_o); + *target_local = PyStackRef_FromPyObjectSteal(temp); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (PyStackRef_IsNull(*target_local)) goto pop_2_error; + #if TIER_ONE + // The STORE_FAST is already done. This is done here in tier one, + // and during trace projection in tier two: + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); + #endif + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_FLOAT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); - } - /* Skip 1 cache entry */ - // _BINARY_OP_MULTIPLY_FLOAT { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval * - ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - res = PyStackRef_FromPyObjectSteal(res_o); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_FLOAT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_MULTIPLY_FLOAT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_INT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); - } - /* Skip 1 cache entry */ - // _BINARY_OP_MULTIPLY_INT { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - res = PyStackRef_FromPyObjectSteal(res_o); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_INT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_MULTIPLY_INT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_FLOAT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); - } - /* Skip 1 cache entry */ - // _BINARY_OP_SUBTRACT_FLOAT { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval - - ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - res = PyStackRef_FromPyObjectSteal(res_o); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_FLOAT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_SUBTRACT_FLOAT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_INT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); - } - /* Skip 1 cache entry */ - // _BINARY_OP_SUBTRACT_INT { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - res = PyStackRef_FromPyObjectSteal(res_o); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_INT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + } + /* Skip 1 cache entry */ + // _BINARY_OP_SUBTRACT_INT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BINARY_SLICE); - _PyStackRef container; - _PyStackRef start; - _PyStackRef stop; - _PyStackRef res; - // _SPECIALIZE_BINARY_SLICE - { - // Placeholder until we implement BINARY_SLICE specialization - #if ENABLE_SPECIALIZATION - OPCODE_DEFERRED_INC(BINARY_SLICE); - #endif /* ENABLE_SPECIALIZATION */ - } - // _BINARY_SLICE { - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyObject *res_o; - // Can't use ERROR_IF() here, because we haven't - // DECREF'ed container yet, and we still own slice. - if (slice == NULL) { - res_o = NULL; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BINARY_SLICE); + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + _PyStackRef res; + // _SPECIALIZE_BINARY_SLICE + { + // Placeholder until we implement BINARY_SLICE specialization + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(BINARY_SLICE); + #endif /* ENABLE_SPECIALIZATION */ } - else { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + // _BINARY_SLICE + { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; _PyFrame_SetStackPointer(frame, stack_pointer); - res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); stack_pointer = _PyFrame_GetStackPointer(frame); - Py_DECREF(slice); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + PyObject *res_o; + // Can't use ERROR_IF() here, because we haven't + // DECREF'ed container yet, and we still own slice. + if (slice == NULL) { + res_o = NULL; + } + else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(slice); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } + PyStackRef_CLOSE(container); + if (res_o == NULL) goto pop_3_error; + res = PyStackRef_FromPyObjectSteal(res_o); } - PyStackRef_CLOSE(container); - if (res_o == NULL) CEVAL_GOTO(pop_3_error); - res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-3] = res; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR); - PREDICTED(BINARY_SUBSCR); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef container; - _PyStackRef sub; - _PyStackRef res; - // _SPECIALIZE_BINARY_SUBSCR { - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - assert(frame->stackpointer == NULL); - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR); + PREDICTED(BINARY_SUBSCR); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef container; + _PyStackRef sub; + _PyStackRef res; + // _SPECIALIZE_BINARY_SUBSCR + { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + assert(frame->stackpointer == NULL); + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_BinarySubscr(container, sub, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(BINARY_SUBSCR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _BINARY_SUBSCR + { + PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); + PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_BinarySubscr(container, sub, next_instr); + PyObject *res_o = PyObject_GetItem(container_o, sub_o); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); } - OPCODE_DEFERRED_INC(BINARY_SUBSCR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _BINARY_SUBSCR - { - PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); - PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_GetItem(container_o, sub_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_DICT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef dict_st; - _PyStackRef sub_st; - _PyStackRef res; - /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - dict_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - PyObject *res_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int rc = PyDict_GetItemRef(dict, sub, &res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (rc == 0) { + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef dict_st; + _PyStackRef sub_st; + _PyStackRef res; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + dict_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res_o; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetKeyError(sub); + int rc = PyDict_GetItemRef(dict, sub, &res_o); stack_pointer = _PyFrame_GetStackPointer(frame); + if (rc == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetKeyError(sub); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + PyStackRef_CLOSE(dict_st); + PyStackRef_CLOSE(sub_st); + if (rc <= 0) goto pop_2_error; + // not found or error + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(dict_st); - PyStackRef_CLOSE(sub_st); - if (rc <= 0) CEVAL_GOTO(pop_2_error); - // not found or error - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef container; - _PyStackRef getitem; - _PyStackRef sub; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); - } - // _BINARY_SUBSCR_CHECK_FUNC + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef container; + _PyStackRef getitem; + _PyStackRef sub; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + } + // _BINARY_SUBSCR_CHECK_FUNC + { + container = stack_pointer[-2]; + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); + DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR); + assert(PyFunction_Check(getitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); + DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR); + PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); + assert(code->co_argcount == 2); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + getitem = PyStackRef_FromPyObjectNew(getitem_o); + STAT_INC(BINARY_SUBSCR, hit); + } + // _BINARY_SUBSCR_INIT_CALL + { + sub = stack_pointer[-1]; + new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + new_frame->localsplus[0] = container; + new_frame->localsplus[1] = sub; + frame->return_offset = 2 ; + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL_PARAMS){ { - container = stack_pointer[-2]; - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR); - assert(PyFunction_Check(getitem_o)); - uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR); - PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); - assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); - getitem = PyStackRef_FromPyObjectNew(getitem_o); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef list_st; + _PyStackRef sub_st; + _PyStackRef res; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyList_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + stack_pointer = _PyFrame_GetStackPointer(frame); + DEOPT_IF(res_o == NULL, BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); - } - // _BINARY_SUBSCR_INIT_CALL - { - sub = stack_pointer[-1]; - new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); - new_frame->localsplus[0] = container; - new_frame->localsplus[1] = sub; - frame->return_offset = 2 ; - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - stack_pointer += -2; + #else + DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + Py_INCREF(res_o); + #endif + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(list_st); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); } DISPATCH(); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef list_st; - _PyStackRef sub_st; - _PyStackRef res; - /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - #ifdef Py_GIL_DISABLED - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); - stack_pointer = _PyFrame_GetStackPointer(frame); - DEOPT_IF(res_o == NULL, BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - #else - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - PyObject *res_o = PyList_GET_ITEM(list, index); - assert(res_o != NULL); - Py_INCREF(res_o); - #endif - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - PyStackRef_CLOSE(list_st); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef str_st; - _PyStackRef sub_st; - _PyStackRef res; - /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - str_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); - // Specialize for reading an ASCII character from any string: - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - PyStackRef_CLOSE(str_st); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef str_st; + _PyStackRef sub_st; + _PyStackRef res; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + str_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + // Specialize for reading an ASCII character from any string: + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(str_st); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef tuple_st; - _PyStackRef sub_st; - _PyStackRef res; - /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - tuple_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - PyObject *res_o = PyTuple_GET_ITEM(tuple, index); - assert(res_o != NULL); - Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - PyStackRef_CLOSE(tuple_st); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + _PyStackRef tuple_st; + _PyStackRef sub_st; + _PyStackRef res; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + tuple_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyTuple_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + Py_INCREF(res_o); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(tuple_st); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_LIST); - _PyStackRef *values; - _PyStackRef list; - values = &stack_pointer[-oparg]; - PyObject *list_o = _PyList_FromStackRefSteal(values, oparg); - if (list_o == NULL) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_LIST); + _PyStackRef *values; + _PyStackRef list; + values = &stack_pointer[-oparg]; + PyObject *list_o = _PyList_FromStackRefSteal(values, oparg); + if (list_o == NULL) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + list = PyStackRef_FromPyObjectSteal(list_o); + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - list = PyStackRef_FromPyObjectSteal(list_o); - stack_pointer[-oparg] = list; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_MAP); - _PyStackRef *values; - _PyStackRef map; - values = &stack_pointer[-oparg*2]; - STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); - if (CONVERSION_FAILED(values_o)) { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_MAP); + _PyStackRef *values; + _PyStackRef map; + values = &stack_pointer[-oparg*2]; + STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); + if (CONVERSION_FAILED(values_o)) { + for (int _i = oparg*2; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + { + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *map_o = _PyDict_FromItems( + values_o, 2, + values_o+1, 2, + oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); for (int _i = oparg*2; --_i >= 0;) { PyStackRef_CLOSE(values[_i]); } - { + if (map_o == NULL) { stack_pointer += -oparg*2; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *map_o = _PyDict_FromItems( - values_o, 2, - values_o+1, 2, - oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); - for (int _i = oparg*2; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); - } - if (map_o == NULL) { - stack_pointer += -oparg*2; + map = PyStackRef_FromPyObjectSteal(map_o); + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); } - map = PyStackRef_FromPyObjectSteal(map_o); - stack_pointer[-oparg*2] = map; - stack_pointer += 1 - oparg*2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SET); - _PyStackRef *values; - _PyStackRef set; - values = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *set_o = PySet_New(NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (set_o == NULL) { - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); - } - { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SET); + _PyStackRef *values; + _PyStackRef set; + values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *set_o = PySet_New(NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (set_o == NULL) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } } - } - int err = 0; - for (int i = 0; i < oparg; i++) { - if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PySet_Add(set_o, PyStackRef_AsPyObjectBorrow(values[i])); - stack_pointer = _PyFrame_GetStackPointer(frame); + int err = 0; + for (int i = 0; i < oparg; i++) { + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PySet_Add(set_o, PyStackRef_AsPyObjectBorrow(values[i])); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + PyStackRef_CLOSE(values[i]); } - PyStackRef_CLOSE(values[i]); - } - if (err != 0) { - Py_DECREF(set_o); - { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + if (err != 0) { + Py_DECREF(set_o); + { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } } + set = PyStackRef_FromPyObjectSteal(set_o); + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - set = PyStackRef_FromPyObjectSteal(set_o); - stack_pointer[-oparg] = set; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SLICE); - _PyStackRef start; - _PyStackRef stop; - _PyStackRef step = PyStackRef_NULL; - _PyStackRef slice; - if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } - stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; - start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; - PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); - PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); - PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); - PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); - PyStackRef_CLOSE(start); - PyStackRef_CLOSE(stop); - PyStackRef_XCLOSE(step); - if (slice_o == NULL) { - stack_pointer += -2 - ((oparg == 3) ? 1 : 0); - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - slice = PyStackRef_FromPyObjectSteal(slice_o); - stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; - stack_pointer += -1 - ((oparg == 3) ? 1 : 0); - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SLICE); + _PyStackRef start; + _PyStackRef stop; + _PyStackRef step = PyStackRef_NULL; + _PyStackRef slice; + if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } + stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; + start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; + PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); + PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); + PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); + PyStackRef_CLOSE(start); + PyStackRef_CLOSE(stop); + PyStackRef_XCLOSE(step); + if (slice_o == NULL) { + stack_pointer += -2 - ((oparg == 3) ? 1 : 0); + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + slice = PyStackRef_FromPyObjectSteal(slice_o); + stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; + stack_pointer += -1 - ((oparg == 3) ? 1 : 0); + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_STRING); - _PyStackRef *pieces; - _PyStackRef str; - pieces = &stack_pointer[-oparg]; - STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); - if (CONVERSION_FAILED(pieces_o)) { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_STRING); + _PyStackRef *pieces; + _PyStackRef str; + pieces = &stack_pointer[-oparg]; + STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); + if (CONVERSION_FAILED(pieces_o)) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(pieces[_i]); + } + { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(pieces[_i]); } - { + if (str_o == NULL) { stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } - } - PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(pieces[_i]); - } - if (str_o == NULL) { - stack_pointer += -oparg; + str = PyStackRef_FromPyObjectSteal(str_o); + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); } - str = PyStackRef_FromPyObjectSteal(str_o); - stack_pointer[-oparg] = str; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_TUPLE); - _PyStackRef *values; - _PyStackRef tup; - values = &stack_pointer[-oparg]; - PyObject *tup_o = _PyTuple_FromStackRefSteal(values, oparg); - if (tup_o == NULL) { - stack_pointer += -oparg; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TUPLE); + _PyStackRef *values; + _PyStackRef tup; + values = &stack_pointer[-oparg]; + PyObject *tup_o = _PyTuple_FromStackRefSteal(values, oparg); + if (tup_o == NULL) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + tup = PyStackRef_FromPyObjectSteal(tup_o); + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); } - tup = PyStackRef_FromPyObjectSteal(tup_o); - stack_pointer[-oparg] = tup; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CACHE); - assert(0 && "Executing a cache."); - Py_FatalError("Executing a cache."); - DISPATCH(); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CACHE); + assert(0 && "Executing a cache."); + Py_FatalError("Executing a cache."); + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL); - PREDICTED(CALL); - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef *func; - _PyStackRef *maybe_self; - _PyStackRef res; - // _SPECIALIZE_CALL { - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL); + PREDICTED(CALL); + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef res; + // _SPECIALIZE_CALL + { + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CALL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD + { + args = &stack_pointer[-oparg]; + func = &stack_pointer[-2 - oparg]; + maybe_self = &stack_pointer[-1 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); + } + } + // _DO_CALL + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; + } + frame->return_offset = 4 ; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable[0]); + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(res_o); + } + } + } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } } - OPCODE_DEFERRED_INC(CALL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS){ { - args = &stack_pointer[-oparg]; - func = &stack_pointer[-2 - oparg]; - maybe_self = &stack_pointer[-1 - oparg]; - if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef *args; + _PyStackRef *init; + _PyStackRef *self; + _PyInterpreterFrame *init_frame; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_AND_ALLOCATE_OBJECT + { + args = &stack_pointer[-oparg]; + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + init = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; + uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL); + PyTypeObject *tp = (PyTypeObject *)callable_o; + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL); + assert(tp->tp_new == PyBaseObject_Type.tp_new); + assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_alloc == PyType_GenericAlloc); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; + PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); + PyCodeObject *code = (PyCodeObject *)init_func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *self_o = PyType_GenericAlloc(tp, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (self_o == NULL) { + goto error; + } + self[0] = PyStackRef_FromPyObjectSteal(self_o); _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); + init[0] = PyStackRef_FromPyObjectNew(init_func); PyStackRef_CLOSE(temp); } - } - // _DO_CALL + // _CREATE_INIT_FRAME + { + args = &stack_pointer[-oparg]; + self = &stack_pointer[-1 - oparg]; + init = &stack_pointer[-2 - oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); + stack_pointer = _PyFrame_GetStackPointer(frame); + /* Push self onto stack of shim */ + shim->localsplus[0] = PyStackRef_DUP(self[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, init[0], NULL, args-1, oparg+1, NULL, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + _PyEval_FrameClearAndPop(tstate, shim); + goto error; + } + init_frame = temp; + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; + /* Account for pushing the extra frame. + * We don't check recursion depth here, + * as it will be checked after start_frame */ + tstate->py_recursion_remaining--; + } + // _PUSH_FRAME + { + new_frame = init_frame; + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS){ { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef *func; + _PyStackRef *self; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); } - // Check if the call can be inlined or not - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + { + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL); + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + func = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + STAT_INC(CALL, hit); + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + PyStackRef_CLOSE(temp); + } + // flush + // _CHECK_FUNCTION_VERSION + { + callable = &stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL); + } + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null[0]); + STAT_INC(CALL, hit); + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null[0]; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS){ + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef *method; + _PyStackRef *self; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_METHOD_VERSION + { + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL); + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + DEOPT_IF(!PyFunction_Check(func), CALL); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + } + // _EXPAND_METHOD + { + method = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + assert(PyStackRef_IsNull(null[0])); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable[0]; + method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(method[0])); + PyStackRef_CLOSE(temp); + } + // flush + // _PY_FRAME_GENERAL { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable[0], locals, args, total_args, NULL, frame ); stack_pointer = _PyFrame_GetStackPointer(frame); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). + // The frame has stolen all the arguments from the stack. stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - CEVAL_GOTO(error); + if (temp == NULL) { + goto error; } - frame->return_offset = 4 ; - DISPATCH_INLINED(new_frame); + new_frame = temp; + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS){ + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_CLASS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_CLASS + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyType_Check(callable_o), CALL); + PyTypeObject *tp = (PyTypeObject *)callable_o; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + STAT_INC(CALL, hit); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); + PyStackRef_CLOSE(arguments[i]); } PyStackRef_CLOSE(callable[0]); - { + if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); - if (res_o == NULL) { + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); + int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(res_o); - } - } - } - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); } - res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - // _CHECK_PERIODIC + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS){ { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* Builtin METH_FASTCALL functions, without keywords */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + /* res = func(self, args, nargs) */ + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); + PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable_o), + args_o, + total_args); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(arguments[i]); + } + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *null; - _PyStackRef *args; - _PyStackRef *init; - _PyStackRef *self; - _PyInterpreterFrame *init_frame; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_AND_ALLOCATE_OBJECT +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS){ { - args = &stack_pointer[-oparg]; - null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - init = &stack_pointer[-2 - oparg]; - self = &stack_pointer[-1 - oparg]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(!PyType_Check(callable_o), CALL); - PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL); - assert(tp->tp_new == PyBaseObject_Type.tp_new); - assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); - assert(tp->tp_alloc == PyType_GenericAlloc); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; - PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); - PyCodeObject *code = (PyCodeObject *)init_func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *self_o = PyType_GenericAlloc(tp, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (self_o == NULL) { - CEVAL_GOTO(error); + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); + STAT_INC(CALL, hit); + /* res = func(self, args, nargs, kwnames) */ + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); } - self[0] = PyStackRef_FromPyObjectSteal(self_o); - _PyStackRef temp = callable[0]; - init[0] = PyStackRef_FromPyObjectNew(init_func); - PyStackRef_CLOSE(temp); - } - // _CREATE_INIT_FRAME - { - args = &stack_pointer[-oparg]; - self = &stack_pointer[-1 - oparg]; - init = &stack_pointer[-2 - oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); - assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); - assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); - stack_pointer = _PyFrame_GetStackPointer(frame); - /* Push self onto stack of shim */ - shim->localsplus[0] = PyStackRef_DUP(self[0]); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, init[0], NULL, args-1, oparg+1, NULL, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - _PyEval_FrameClearAndPop(tstate, shim); - CEVAL_GOTO(error); + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } } - init_frame = temp; - frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; - /* Account for pushing the extra frame. - * We don't check recursion depth here, - * as it will be checked after start_frame */ - tstate->py_recursion_remaining--; - } - // _PUSH_FRAME - { - new_frame = init_frame; - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *null; - _PyStackRef *func; - _PyStackRef *self; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS - { - null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL); - } - // _INIT_CALL_BOUND_METHOD_EXACT_ARGS - { - func = &stack_pointer[-2 - oparg]; - self = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - STAT_INC(CALL, hit); - self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - PyStackRef_CLOSE(temp); - } - // flush - // _CHECK_FUNCTION_VERSION - { - callable = &stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); - } - // _CHECK_FUNCTION_EXACT_ARGS - { - self_or_null = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - assert(PyFunction_Check(callable_o)); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); - } - // _CHECK_STACK_SPACE - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); - } - // _INIT_CALL_PY_EXACT_ARGS +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS){ { - args = &stack_pointer[-oparg]; - int has_self = !PyStackRef_IsNull(self_or_null[0]); - STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null[0]; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_O + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* Builtin METH_O functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + _PyStackRef arg = args[0]; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(arg); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); } - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - stack_pointer += -2 - oparg; + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *null; - _PyStackRef *method; - _PyStackRef *self; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_METHOD_VERSION - { - null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL); - PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - } - // _EXPAND_METHOD - { - method = &stack_pointer[-2 - oparg]; - self = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - assert(PyStackRef_IsNull(null[0])); - assert(Py_TYPE(callable_o) == &PyMethod_Type); - self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - _PyStackRef temp = callable[0]; - method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(method[0])); - PyStackRef_CLOSE(temp); - } - // flush - // _PY_FRAME_GENERAL +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS){ { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_FUNCTION_EX); + PREDICTED(CALL_FUNCTION_EX); + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef kwargs_in = PyStackRef_NULL; + _PyStackRef tuple; + _PyStackRef kwargs_out = PyStackRef_NULL; + _PyStackRef func_st; + _PyStackRef callargs_st; + _PyStackRef kwargs_st = PyStackRef_NULL; + _PyStackRef result; + // _MAKE_CALLARGS_A_TUPLE + { + if (oparg & 1) { kwargs_in = stack_pointer[-(oparg & 1)]; } + callargs = stack_pointer[-1 - (oparg & 1)]; + func = stack_pointer[-3 - (oparg & 1)]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (PyTuple_CheckExact(callargs_o)) { + tuple = callargs; + kwargs_out = kwargs_in; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { + goto error; + } + kwargs_out = kwargs_in; + PyStackRef_CLOSE(callargs); + tuple = PyStackRef_FromPyObjectSteal(tuple_o); + } } - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - args, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - // The frame has stolen all the arguments from the stack. - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - CEVAL_GOTO(error); + // _DO_CALL_FUNCTION_EX + { + kwargs_st = kwargs_out; + callargs_st = tuple; + func_st = func; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-1 - (oparg & 1)] = callargs_st; + if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(result_o); + } + } + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer += -2 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Need to sync the stack since we exit with DISPATCH_INLINED. + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + goto error; + } + assert( 1 == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + stack_pointer[-1 - (oparg & 1)] = callargs_st; + if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(kwargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(callargs_st); + PyStackRef_CLOSE(func_st); + if (result_o == NULL) { + stack_pointer += -3 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + result = PyStackRef_FromPyObjectSteal(result_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3 - (oparg & 1)] = result; + stack_pointer += -2 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 2 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + } } - new_frame = temp; + stack_pointer[-3 - (oparg & 1)] = result; + stack_pointer += -2 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } - // _SAVE_RETURN_OFFSET + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS){ { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_1); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(oparg <= MAX_INTRINSIC_1); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) goto pop_1_error; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-1] = res; + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS){ { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_2); + _PyStackRef value2_st; + _PyStackRef value1_st; + _PyStackRef res; + value1_st = stack_pointer[-1]; + value2_st = stack_pointer[-2]; + assert(oparg <= MAX_INTRINSIC_2); + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value2_st); + PyStackRef_CLOSE(value1_st); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_CLASS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_CLASS +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS){ { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ISINSTANCE); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; + /* isinstance(o, o2) */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyType_Check(callable_o), CALL); - PyTypeObject *tp = (PyTypeObject *)callable_o; int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { arguments--; total_args++; } - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + DEOPT_IF(total_args != 2, CALL); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL); STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - } + _PyStackRef cls_stackref = arguments[1]; + _PyStackRef inst_stackref = arguments[0]; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(arguments[i]); + if (retval < 0) { + goto error; } + res = retval ? PyStackRef_True : PyStackRef_False; + assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL functions, without keywords */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW); + PREDICTED(CALL_KW); + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef kwnames_in; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef kwnames_out; + _PyStackRef res; + // _SPECIALIZE_CALL_KW + { + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CALL_KW); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - /* res = func(self, args, nargs) */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + kwnames_in = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + func = &stack_pointer[-3 - oparg]; + maybe_self = &stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); } + kwnames_out = kwnames_in; + } + // _DO_CALL_KW + { + kwnames = kwnames_out; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) { - stack_pointer += -2 - oparg; + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-1] = kwnames; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(kwnames); + // Sync stack explicitly since we leave using DISPATCH_INLINED(). + stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; + } + assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4 ; + DISPATCH_INLINED(new_frame); } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + stack_pointer[-1] = kwnames; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(res_o); + } + } + } + PyStackRef_CLOSE(kwnames); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable_o), - args_o, - total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(arguments[i]); - } - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - // _CHECK_PERIODIC + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS){ { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef kwnames; + _PyStackRef *method; + _PyStackRef *self; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + } + // _CHECK_METHOD_VERSION_KW + { + null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW); + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + DEOPT_IF(!PyFunction_Check(func), CALL_KW); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW); + } + // _EXPAND_METHOD_KW + { + method = &stack_pointer[-3 - oparg]; + self = &stack_pointer[-2 - oparg]; + _PyStackRef callable_s = callable[0]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable_s); + assert(PyStackRef_IsNull(null[0])); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(method[0])); + PyStackRef_CLOSE(callable_s); + } + // flush + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, positional_args, kwnames_o, frame + ); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 1 + oparg; + PyStackRef_CLOSE(kwnames); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + goto error; + } + new_frame = temp; } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST_WITH_KEYWORDS +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS){ { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_NON_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef kwnames; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE_KW + { + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(PyFunction_Check(callable_o), CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW); } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); - STAT_INC(CALL, hit); - /* res = func(self, args, nargs, kwnames) */ - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { + // _CALL_KW_NON_PY + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(kwnames); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + if (res_o == NULL) { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } - { + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 2 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - // _CHECK_PERIODIC + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef kwnames; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + } + // _CHECK_FUNCTION_VERSION_KW + { + callable = &stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL_KW); + } + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, positional_args, kwnames_o, frame + ); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 1 + oparg; + PyStackRef_CLOSE(kwnames); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + goto error; + } + new_frame = temp; } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_O); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_O +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LEN); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - /* Builtin METH_O functions */ + /* len(o) */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -1764,1456 +3228,942 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS) total_args++; } DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL); - // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable_o != interp->callable_cache.len, CALL); STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - _PyStackRef arg = args[0]; - _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyStackRef arg_stackref = args[0]; + PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); + Py_ssize_t len_i = PyObject_Length(arg); stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); + if (len_i < 0) { + goto error; + } + PyObject *res_o = PyLong_FromSsize_t(len_i); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(arg); - PyStackRef_CLOSE(callable[0]); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } + PyStackRef_CLOSE(arg_stackref); + PyStackRef_CLOSE(callable[0]); res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - // _CHECK_PERIODIC + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LIST_APPEND); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self; + _PyStackRef arg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + arg = stack_pointer[-1]; + self = stack_pointer[-2]; + callable = stack_pointer[-3]; + assert(oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL); + assert(self_o != NULL); + DEOPT_IF(!PyList_Check(self_o), CALL); + DEOPT_IF(!LOCK_OBJECT(self_o), CALL); + STAT_INC(CALL, hit); + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); + PyStackRef_CLOSE(self); + PyStackRef_CLOSE(callable); + if (err) goto pop_3_error; + #if TIER_ONE + // Skip the following POP_TOP. This is done here in tier one, and + // during trace projection in tier two: + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); + #endif + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_FUNCTION_EX); - PREDICTED(CALL_FUNCTION_EX); - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; - _PyStackRef func; - _PyStackRef callargs; - _PyStackRef kwargs_in = PyStackRef_NULL; - _PyStackRef tuple; - _PyStackRef kwargs_out = PyStackRef_NULL; - _PyStackRef func_st; - _PyStackRef callargs_st; - _PyStackRef kwargs_st = PyStackRef_NULL; - _PyStackRef result; - // _MAKE_CALLARGS_A_TUPLE - { - if (oparg & 1) { kwargs_in = stack_pointer[-(oparg & 1)]; } - callargs = stack_pointer[-1 - (oparg & 1)]; - func = stack_pointer[-3 - (oparg & 1)]; - PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); - if (PyTuple_CheckExact(callargs_o)) { - tuple = callargs; - kwargs_out = kwargs_in; - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - CEVAL_GOTO(error); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS){ + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + /* Builtin METH_FASTCALL methods, without keywords */ + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + int nargs = total_args - 1; + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *tuple_o = PySequence_Tuple(callargs_o); + PyCFunctionFast cfunc = + (PyCFunctionFast)(void(*)(void))meth->ml_meth; + PyObject *res_o = cfunc(self, (args_o + 1), nargs); stack_pointer = _PyFrame_GetStackPointer(frame); - if (tuple_o == NULL) { - CEVAL_GOTO(error); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Clear the stack of the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } - kwargs_out = kwargs_in; - PyStackRef_CLOSE(callargs); - tuple = PyStackRef_FromPyObjectSteal(tuple_o); + res = PyStackRef_FromPyObjectSteal(res_o); } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - // _DO_CALL_FUNCTION_EX + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS){ { - kwargs_st = kwargs_out; - callargs_st = tuple; - func_st = func; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - PyObject *result_o; - assert(!_PyErr_Occurred(tstate)); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - assert(PyTuple_CheckExact(callargs)); - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - stack_pointer[-1 - (oparg & 1)] = callargs_st; - if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - CEVAL_GOTO(error); + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; } - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + PyTypeObject *d_type = method->d_common.d_type; + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + STAT_INC(CALL, hit); + int nargs = total_args - 1; + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(result_o); - } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -2 - (oparg & 1); + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( - tstate, func_st, locals, - nargs, callargs, kwargs, frame); + int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - // Need to sync the stack since we exit with DISPATCH_INLINED. - stack_pointer += -1; + if (err != 0) goto error; + stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { - CEVAL_GOTO(error); - } - assert( 1 == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); } - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[-1 - (oparg & 1)] = callargs_st; - if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(callargs_st); - PyStackRef_CLOSE(func_st); - if (result_o == NULL) { - stack_pointer += -3 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); } - result = PyStackRef_FromPyObjectSteal(result_o); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - // _CHECK_PERIODIC + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS){ { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-3 - (oparg & 1)] = result; - stack_pointer += -2 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_NOARGS + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + assert(oparg == 0 || oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _Py_EnterRecursiveCallTstateUnchecked(tstate); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 2 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-3 - (oparg & 1)] = result; - stack_pointer += -2 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_1); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - assert(oparg <= MAX_INTRINSIC_1); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (res_o == NULL) CEVAL_GOTO(pop_1_error); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; - DISPATCH(); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_2); - _PyStackRef value2_st; - _PyStackRef value1_st; - _PyStackRef res; - value1_st = stack_pointer[-1]; - value2_st = stack_pointer[-2]; - assert(oparg <= MAX_INTRINSIC_2); - PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); - PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value2_st); - PyStackRef_CLOSE(value1_st); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ISINSTANCE); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - /* isinstance(o, o2) */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - DEOPT_IF(total_args != 2, CALL); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL); - STAT_INC(CALL, hit); - _PyStackRef cls_stackref = arguments[1]; - _PyStackRef inst_stackref = arguments[0]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (retval < 0) { - CEVAL_GOTO(error); - } - res = retval ? PyStackRef_True : PyStackRef_False; - assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW); - PREDICTED(CALL_KW); - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef kwnames_in; - _PyStackRef *func; - _PyStackRef *maybe_self; - _PyStackRef kwnames_out; - _PyStackRef res; - // _SPECIALIZE_CALL_KW - { - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS){ + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_O + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_O, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + _PyStackRef arg_stackref = args[1]; + _PyStackRef self_stackref = args[0]; + DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _Py_EnterRecursiveCallTstateUnchecked(tstate); _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, + PyStackRef_AsPyObjectBorrow(self_stackref), + PyStackRef_AsPyObjectBorrow(arg_stackref)); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(arg_stackref); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); } - OPCODE_DEFERRED_INC(CALL_KW); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD_KW - { - kwnames_in = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - func = &stack_pointer[-3 - oparg]; - maybe_self = &stack_pointer[-2 - oparg]; - if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); - PyStackRef_CLOSE(temp); + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } } - kwnames_out = kwnames_in; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - // _DO_CALL_KW + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS){ { - kwnames = kwnames_out; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_NON_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE + { + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(PyFunction_Check(callable_o), CALL); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL); } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - // Check if the call can be inlined or not - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + // _CALL_NON_PY_GENERAL { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-1] = kwnames; + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - args, positional_args, kwnames_o, frame - ); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(kwnames); - // Sync stack explicitly since we leave using DISPATCH_INLINED(). - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - CEVAL_GOTO(error); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); } - assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); - frame->return_offset = 4 ; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - PyStackRef_CLOSE(kwnames); - { - stack_pointer += -3 - oparg; + if (res_o == NULL) { + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[-1] = kwnames; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); + int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(res_o); - } + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } - PyStackRef_CLOSE(kwnames); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *null; - _PyStackRef kwnames; - _PyStackRef *method; - _PyStackRef *self; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW); - } - // _CHECK_METHOD_VERSION_KW - { - null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW); - PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL_KW); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW); - } - // _EXPAND_METHOD_KW - { - method = &stack_pointer[-3 - oparg]; - self = &stack_pointer[-2 - oparg]; - _PyStackRef callable_s = callable[0]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable_s); - assert(PyStackRef_IsNull(null[0])); - assert(Py_TYPE(callable_o) == &PyMethod_Type); - self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(method[0])); - PyStackRef_CLOSE(callable_s); - } - // flush - // _PY_FRAME_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - args, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(kwnames); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - stack_pointer += -3 - oparg; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - CEVAL_GOTO(error); - } - new_frame = temp; - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_NON_PY); - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef kwnames; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CHECK_IS_NOT_PY_CALLABLE_KW - { - callable = &stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL_KW); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW); - } - // _CALL_KW_NON_PY +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS){ { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - #if TIER_ONE - assert(opcode != INSTRUMENTED_CALL); - #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - PyStackRef_CLOSE(kwnames); - { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } + // _CHECK_FUNCTION_VERSION + { + callable = &stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL); } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(kwnames); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 2 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_PY); - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef kwnames; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW); - } - // _CHECK_FUNCTION_VERSION_KW - { - callable = &stack_pointer[-3 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL_KW); - } - // _PY_FRAME_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - args, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(kwnames); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - CEVAL_GOTO(error); + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); } - new_frame = temp; - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - DISPATCH(); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LEN); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - /* len(o) */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.len, CALL); - STAT_INC(CALL, hit); - _PyStackRef arg_stackref = args[0]; - PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_ssize_t len_i = PyObject_Length(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) { - CEVAL_GOTO(error); - } - PyObject *res_o = PyLong_FromSsize_t(len_i); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res_o == NULL) { - CEVAL_GOTO(error); - } - PyStackRef_CLOSE(arg_stackref); - PyStackRef_CLOSE(callable[0]); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LIST_APPEND); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self; - _PyStackRef arg; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - arg = stack_pointer[-1]; - self = stack_pointer[-2]; - callable = stack_pointer[-3]; - assert(oparg == 1); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL); - assert(self_o != NULL); - DEOPT_IF(!PyList_Check(self_o), CALL); - DEOPT_IF(!LOCK_OBJECT(self_o), CALL); - STAT_INC(CALL, hit); - int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); - UNLOCK_OBJECT(self_o); - PyStackRef_CLOSE(self); - PyStackRef_CLOSE(callable); - if (err) CEVAL_GOTO(pop_3_error); - #if TIER_ONE - // Skip the following POP_TOP. This is done here in tier one, and - // during trace projection in tier two: - assert(next_instr->op.code == POP_TOP); - SKIP_OVER(1); - #endif - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); - PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null[0]); + STAT_INC(CALL, hit); + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null[0]; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFast cfunc = - (PyCFunctionFast)(void(*)(void))meth->ml_meth; - PyObject *res_o = cfunc(self, (args_o + 1), nargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS){ { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); - PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); - STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); } - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + // _CHECK_FUNCTION_VERSION + { + callable = &stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL); } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, total_args, NULL, frame + ); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_NOARGS - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - _PyStackRef self_stackref = args[0]; - PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); - // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - _Py_EnterRecursiveCallTstateUnchecked(tstate); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(self_stackref); - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { + // The frame has stolen all the arguments from the stack. stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + if (temp == NULL) { + goto error; + } + new_frame = temp; } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_O - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); - // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); - _PyStackRef arg_stackref = args[1]; - _PyStackRef self_stackref = args[0]; - DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - _Py_EnterRecursiveCallTstateUnchecked(tstate); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, - PyStackRef_AsPyObjectBorrow(self_stackref), - PyStackRef_AsPyObjectBorrow(arg_stackref)); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(self_stackref); - PyStackRef_CLOSE(arg_stackref); - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS){ { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_STR_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_STR_1 + { + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL); + STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); + PyObject *res_o = PyObject_Str(arg_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_NON_PY_GENERAL); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CHECK_IS_NOT_PY_CALLABLE - { - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL); - } - // _CALL_NON_PY_GENERAL - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - #if TIER_ONE - assert(opcode != INSTRUMENTED_CALL); - #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; + PyStackRef_CLOSE(arg); + if (res_o == NULL) goto pop_3_error; + res = PyStackRef_FromPyObjectSteal(res_o); } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - { - stack_pointer += -2 - oparg; + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); } } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_FUNCTION_VERSION - { - callable = &stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); - } - // _CHECK_FUNCTION_EXACT_ARGS - { - self_or_null = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - assert(PyFunction_Check(callable_o)); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); - } - // _CHECK_STACK_SPACE - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); - } - // _INIT_CALL_PY_EXACT_ARGS - { - args = &stack_pointer[-oparg]; - int has_self = !PyStackRef_IsNull(self_or_null[0]); - STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null[0]; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; - } - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - DISPATCH(); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_GENERAL); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_FUNCTION_VERSION - { - callable = &stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); - } - // _PY_FRAME_GENERAL - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - args, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - // The frame has stolen all the arguments from the stack. - stack_pointer += -2 - oparg; + stack_pointer[-3] = res; + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - CEVAL_GOTO(error); - } - new_frame = temp; - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_STR_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_STR_1 - { - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL); - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Str(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(arg); - if (res_o == NULL) CEVAL_GOTO(pop_3_error); - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-3] = res; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TUPLE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_TUPLE_1 + { + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL); + STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); + PyObject *res_o = PySequence_Tuple(arg_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + PyStackRef_CLOSE(arg); + if (res_o == NULL) goto pop_3_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } } + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-3] = res; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TUPLE_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_TUPLE_1 +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS){ { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TYPE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ arg = stack_pointer[-1]; null = stack_pointer[-2]; callable = stack_pointer[-3]; @@ -3221,5278 +4171,8272 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL); + DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL); STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PySequence_Tuple(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); PyStackRef_CLOSE(arg); - if (res_o == NULL) CEVAL_GOTO(pop_3_error); - res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } - // _CHECK_PERIODIC + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS){ { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-3] = res; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EG_MATCH); + _PyStackRef exc_value_st; + _PyStackRef match_type_st; + _PyStackRef rest; + _PyStackRef match; + match_type_st = stack_pointer[-1]; + exc_value_st = stack_pointer[-2]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + PyStackRef_CLOSE(exc_value_st); + PyStackRef_CLOSE(match_type_st); + goto pop_2_error; + } + PyObject *match_o = NULL; + PyObject *rest_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, + &match_o, &rest_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(exc_value_st); + PyStackRef_CLOSE(match_type_st); + if (res < 0) goto pop_2_error; + assert((match_o == NULL) == (rest_o == NULL)); + if (match_o == NULL) goto pop_2_error; + if (!Py_IsNone(match_o)) { stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); + PyErr_SetHandledException(match_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); } - } - stack_pointer[-3] = res; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TYPE_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL); - STAT_INC(CALL, hit); - res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); - PyStackRef_CLOSE(arg); - stack_pointer[-3] = res; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + rest = PyStackRef_FromPyObjectSteal(rest_o); + match = PyStackRef_FromPyObjectSteal(match_o); + stack_pointer[-2] = rest; + stack_pointer[-1] = match; + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EG_MATCH); - _PyStackRef exc_value_st; - _PyStackRef match_type_st; - _PyStackRef rest; - _PyStackRef match; - match_type_st = stack_pointer[-1]; - exc_value_st = stack_pointer[-2]; - PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - PyStackRef_CLOSE(exc_value_st); - PyStackRef_CLOSE(match_type_st); - CEVAL_GOTO(pop_2_error); - } - PyObject *match_o = NULL; - PyObject *rest_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match_o, &rest_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(exc_value_st); - PyStackRef_CLOSE(match_type_st); - if (res < 0) CEVAL_GOTO(pop_2_error); - assert((match_o == NULL) == (rest_o == NULL)); - if (match_o == NULL) CEVAL_GOTO(pop_2_error); - if (!Py_IsNone(match_o)) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS){ + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EXC_MATCH); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyExceptionInstance_Check(left_o)); _PyFrame_SetStackPointer(frame, stack_pointer); - PyErr_SetHandledException(match_o); + int err = _PyEval_CheckExceptTypeValid(tstate, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + PyStackRef_CLOSE(right); + goto pop_1_error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyErr_GivenExceptionMatches(left_o, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - } - rest = PyStackRef_FromPyObjectSteal(rest_o); - match = PyStackRef_FromPyObjectSteal(match_o); - stack_pointer[-2] = rest; - stack_pointer[-1] = match; - DISPATCH(); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EXC_MATCH); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyExceptionInstance_Check(left_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyEval_CheckExceptTypeValid(tstate, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { PyStackRef_CLOSE(right); - CEVAL_GOTO(pop_1_error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PyErr_GivenExceptionMatches(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(right); - b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[-1] = b; - DISPATCH(); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[-1] = b; + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(CLEANUP_THROW); - _PyStackRef sub_iter_st; - _PyStackRef last_sent_val_st; - _PyStackRef exc_value_st; - _PyStackRef none; - _PyStackRef value; - exc_value_st = stack_pointer[-1]; - last_sent_val_st = stack_pointer[-2]; - sub_iter_st = stack_pointer[-3]; - PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - #ifndef Py_TAIL_CALL_INTERP - assert(throwflag); - #endif - assert(exc_value && PyExceptionInstance_Check(exc_value)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - none = PyStackRef_None; - value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); - PyStackRef_CLOSE(sub_iter_st); - PyStackRef_CLOSE(last_sent_val_st); - PyStackRef_CLOSE(exc_value_st); - } - else { + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(CLEANUP_THROW); + _PyStackRef sub_iter_st; + _PyStackRef last_sent_val_st; + _PyStackRef exc_value_st; + _PyStackRef none; + _PyStackRef value; + exc_value_st = stack_pointer[-1]; + last_sent_val_st = stack_pointer[-2]; + sub_iter_st = stack_pointer[-3]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + #ifndef Py_TAIL_CALL_INTERP + assert(throwflag); + #endif + assert(exc_value && PyExceptionInstance_Check(exc_value)); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); - monitor_reraise(tstate, frame, this_instr); + int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); stack_pointer = _PyFrame_GetStackPointer(frame); - none = PyStackRef_NULL; - value = PyStackRef_NULL; - CEVAL_GOTO(exception_unwind); + if (matches) { + none = PyStackRef_None; + value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); + PyStackRef_CLOSE(sub_iter_st); + PyStackRef_CLOSE(last_sent_val_st); + PyStackRef_CLOSE(exc_value_st); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + none = PyStackRef_NULL; + value = PyStackRef_NULL; + goto exception_unwind; + } + stack_pointer[-3] = none; + stack_pointer[-2] = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-3] = none; - stack_pointer[-2] = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP); - PREDICTED(COMPARE_OP); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _SPECIALIZE_COMPARE_OP { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP); + PREDICTED(COMPARE_OP); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _SPECIALIZE_COMPARE_OP + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CompareOp(left, right, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(COMPARE_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ } - OPCODE_DEFERRED_INC(COMPARE_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ - } - // _COMPARE_OP - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert((oparg >> 5) <= Py_GE); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - if (oparg & 16) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + // _COMPARE_OP + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert((oparg >> 5) <= Py_GE); _PyFrame_SetStackPointer(frame, stack_pointer); - int res_bool = PyObject_IsTrue(res_o); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); stack_pointer = _PyFrame_GetStackPointer(frame); - Py_DECREF(res_o); - if (res_bool < 0) CEVAL_GOTO(error); - res = res_bool ? PyStackRef_True : PyStackRef_False; - } - else { - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res_o == NULL) goto pop_2_error; + if (oparg & 16) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res_bool = PyObject_IsTrue(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(res_o); + if (res_bool < 0) goto error; + res = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + } } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_FLOAT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP); - } - /* Skip 1 cache entry */ - // _COMPARE_OP_FLOAT { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left_o); - double dright = PyFloat_AS_DOUBLE(right_o); - // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg - int sign_ish = COMPARISON_BIT(dleft, dright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - // It's always a bool, so we don't care about oparg & 16. + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_FLOAT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP); + } + /* Skip 1 cache entry */ + // _COMPARE_OP_FLOAT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); + // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg + int sign_ish = COMPARISON_BIT(dleft, dright); + PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + // It's always a bool, so we don't care about oparg & 16. + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_INT); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_INT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP); - } - /* Skip 1 cache entry */ - // _COMPARE_OP_INT { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_INT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_INT + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP); + } + /* Skip 1 cache entry */ + // _COMPARE_OP_INT + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && _PyLong_DigitCount((PyLongObject *)right_o) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); - // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg - int sign_ish = COMPARISON_BIT(ileft, iright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - // It's always a bool, so we don't care about oparg & 16. + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); + // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg + int sign_ish = COMPARISON_BIT(ileft, iright); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + // It's always a bool, so we don't care about oparg & 16. + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_STR); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_UNICODE - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP); - } - /* Skip 1 cache entry */ - // _COMPARE_OP_STR { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left_o, right_o); - assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(eq == 0 || eq == 1); - assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); - assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; - // It's always a bool, so we don't care about oparg & 16. + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_STR); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_BOTH_UNICODE + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP); + } + /* Skip 1 cache entry */ + // _COMPARE_OP_STR + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left_o, right_o); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; + // It's always a bool, so we don't care about oparg & 16. + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP); - PREDICTED(CONTAINS_OP); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - // _SPECIALIZE_CONTAINS_OP { - right = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP); + PREDICTED(CONTAINS_OP); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + // _SPECIALIZE_CONTAINS_OP + { + right = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ContainsOp(right, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CONTAINS_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CONTAINS_OP + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ContainsOp(right, next_instr); + int res = PySequence_Contains(right_o, left_o); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) goto pop_2_error; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } - OPCODE_DEFERRED_INC(CONTAINS_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - // _CONTAINS_OP + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS){ { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_DICT); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + /* Skip 1 cache entry */ + right = stack_pointer[-1]; left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP); + STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PySequence_Contains(right_o, left_o); + int res = PyDict_Contains(right_o, left_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); - if (res < 0) CEVAL_GOTO(pop_2_error); + if (res < 0) goto pop_2_error; b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = b; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP_DICT); - static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - /* Skip 1 cache entry */ - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP); - STAT_INC(CONTAINS_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PyDict_Contains(right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - if (res < 0) CEVAL_GOTO(pop_2_error); - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP_SET); - static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - /* Skip 1 cache entry */ - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP); - STAT_INC(CONTAINS_OP, hit); - // Note: both set and frozenset use the same seq_contains method! - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PySet_Contains((PySetObject *)right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - if (res < 0) CEVAL_GOTO(pop_2_error); - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_SET); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + /* Skip 1 cache entry */ + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP); + STAT_INC(CONTAINS_OP, hit); + // Note: both set and frozenset use the same seq_contains method! + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PySet_Contains((PySetObject *)right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) goto pop_2_error; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CONVERT_VALUE); - _PyStackRef value; - _PyStackRef result; - value = stack_pointer[-1]; - conversion_func conv_fn; - assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = _PyEval_ConversionFuncs[oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (result_o == NULL) CEVAL_GOTO(pop_1_error); - result = PyStackRef_FromPyObjectSteal(result_o); - stack_pointer[-1] = result; - DISPATCH(); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONVERT_VALUE); + _PyStackRef value; + _PyStackRef result; + value = stack_pointer[-1]; + conversion_func conv_fn; + assert(oparg >= FVC_STR && oparg <= FVC_ASCII); + conv_fn = _PyEval_ConversionFuncs[oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (result_o == NULL) goto pop_1_error; + result = PyStackRef_FromPyObjectSteal(result_o); + stack_pointer[-1] = result; + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY); - _PyStackRef bottom; - _PyStackRef top; - bottom = stack_pointer[-1 - (oparg-1)]; - assert(oparg > 0); - top = PyStackRef_DUP(bottom); - stack_pointer[0] = top; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY); + _PyStackRef bottom; + _PyStackRef top; + bottom = stack_pointer[-1 - (oparg-1)]; + assert(oparg > 0); + top = PyStackRef_DUP(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY_FREE_VARS); - /* Copy closure variables to free variables */ - PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - PyObject *closure = func->func_closure; - assert(oparg == co->co_nfreevars); - int offset = co->co_nlocalsplus - oparg; - for (int i = 0; i < oparg; ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); - } - DISPATCH(); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY_FREE_VARS); + /* Copy closure variables to free variables */ + PyCodeObject *co = _PyFrame_GetCode(frame); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; + assert(oparg == co->co_nfreevars); + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); + } + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_ATTR); - _PyStackRef owner; - owner = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(owner); - if (err) CEVAL_GOTO(pop_1_error); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_ATTR); + _PyStackRef owner; + owner = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(owner); + if (err) goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_DEREF); - PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - // Can't use ERROR_IF here. - // Fortunately we don't need its superpower. - PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); - if (oldobj == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_DEREF); + PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + // Can't use ERROR_IF here. + // Fortunately we don't need its superpower. + PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); + if (oldobj == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + Py_DECREF(oldobj); } - Py_DECREF(oldobj); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_FAST); - _PyStackRef v = GETLOCAL(oparg); - if (PyStackRef_IsNull(v)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_FAST); + _PyStackRef v = GETLOCAL(oparg); + if (PyStackRef_IsNull(v)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + SETLOCAL(oparg, PyStackRef_NULL); } - SETLOCAL(oparg, PyStackRef_NULL); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_GLOBAL); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Pop(GLOBALS(), name, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - // Can't use ERROR_IF here. - if (err < 0) { - CEVAL_GOTO(error); - } - if (err == 0) { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_GLOBAL); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + int err = PyDict_Pop(GLOBALS(), name, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + // Can't use ERROR_IF here. + if (err < 0) { + goto error; + } + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_NAME); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_NAME); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_DelItem(ns, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - // Can't use ERROR_IF here. - if (err != 0) { + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); + err = PyObject_DelItem(ns, name); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + // Can't use ERROR_IF here. + if (err != 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, + name); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_SUBSCR); - _PyStackRef container; - _PyStackRef sub; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - /* del container[sub] */ - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_SUBSCR); + _PyStackRef container; + _PyStackRef sub; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + /* del container[sub] */ + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (err) CEVAL_GOTO(pop_2_error); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (err) goto pop_2_error; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_MERGE); - _PyStackRef callable; - _PyStackRef dict; - _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - callable = stack_pointer[-5 - (oparg - 1)]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_MergeEx(dict_o, update_o, 2); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_MERGE); + _PyStackRef callable; + _PyStackRef dict; + _PyStackRef update; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatKwargsError(tstate, callable_o, update_o); + int err = _PyDict_MergeEx(dict_o, update_o, 2); stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatKwargsError(tstate, callable_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(update); + goto pop_1_error; + } PyStackRef_CLOSE(update); - CEVAL_GOTO(pop_1_error); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(update); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_UPDATE); - _PyStackRef dict; - _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Update(dict_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_UPDATE); + _PyStackRef dict; + _PyStackRef update; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + int err = PyDict_Update(dict_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { + if (err < 0) { _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(update_o)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + PyStackRef_CLOSE(update); + goto pop_1_error; } PyStackRef_CLOSE(update); - CEVAL_GOTO(pop_1_error); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(update); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(END_ASYNC_FOR); - _PyStackRef awaitable_st; - _PyStackRef exc_st; - exc_st = stack_pointer[-1]; - awaitable_st = stack_pointer[-2]; - PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); - assert(exc && PyExceptionInstance_Check(exc)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - PyStackRef_CLOSE(awaitable_st); - PyStackRef_CLOSE(exc_st); - } - else { - Py_INCREF(exc); + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(END_ASYNC_FOR); + _PyStackRef awaitable_st; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; + awaitable_st = stack_pointer[-2]; + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(exc && PyExceptionInstance_Check(exc)); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); + int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(exception_unwind); + if (matches) { + PyStackRef_CLOSE(awaitable_st); + PyStackRef_CLOSE(exc_st); + } + else { + Py_INCREF(exc); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto exception_unwind; + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS){ - next_instr += 1; - INSTRUCTION_STATS(END_FOR); - _PyStackRef value; - value = stack_pointer[-1]; - /* Don't update instr_ptr, so that POP_ITER sees - * the FOR_ITER as the previous instruction. - * This has the benign side effect that if value is - * finalized it will see the location as the FOR_ITER's. - */ - PyStackRef_CLOSE(value); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + { + next_instr += 1; + INSTRUCTION_STATS(END_FOR); + _PyStackRef value; + value = stack_pointer[-1]; + /* Don't update instr_ptr, so that POP_ITER sees + * the FOR_ITER as the previous instruction. + * This has the benign side effect that if value is + * finalized it will see the location as the FOR_ITER's. + */ + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_SEND); - _PyStackRef receiver; - _PyStackRef value; - _PyStackRef val; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - (void)receiver; - val = value; - PyStackRef_CLOSE(receiver); - stack_pointer[-2] = val; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_SEND); + _PyStackRef receiver; + _PyStackRef value; + _PyStackRef val; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + (void)receiver; + val = value; + PyStackRef_CLOSE(receiver); + stack_pointer[-2] = val; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(ENTER_EXECUTOR); - #ifdef _Py_TIER2 - PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; - assert(executor->vm_data.index == INSTR_OFFSET() - 1); - assert(executor->vm_data.code == code); - assert(executor->vm_data.valid); - assert(tstate->previous_executor == NULL); - /* If the eval breaker is set then stay in tier 1. - * This avoids any potentially infinite loops - * involving _RESUME_CHECK */ - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - opcode = executor->vm_data.opcode; - oparg = (oparg & ~255) | executor->vm_data.oparg; - next_instr = this_instr; - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(ENTER_EXECUTOR); + #ifdef _Py_TIER2 + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; + assert(executor->vm_data.index == INSTR_OFFSET() - 1); + assert(executor->vm_data.code == code); + assert(executor->vm_data.valid); + assert(tstate->previous_executor == NULL); + /* If the eval breaker is set then stay in tier 1. + * This avoids any potentially infinite loops + * involving _RESUME_CHECK */ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + opcode = executor->vm_data.opcode; + oparg = (oparg & ~255) | executor->vm_data.oparg; + next_instr = this_instr; + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + DISPATCH_GOTO(); } - DISPATCH_GOTO(); + tstate->previous_executor = Py_None; + Py_INCREF(executor); + GOTO_TIER_TWO(executor); + #else + Py_FatalError("ENTER_EXECUTOR is not supported in this build"); + #endif /* _Py_TIER2 */ } - tstate->previous_executor = Py_None; - Py_INCREF(executor); - GOTO_TIER_TWO(executor); - #else - Py_FatalError("ENTER_EXECUTOR is not supported in this build"); - #endif /* _Py_TIER2 */ DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXIT_INIT_CHECK); - _PyStackRef should_be_none; - should_be_none = stack_pointer[-1]; - assert(STACK_LEVEL() == 2); - if (!PyStackRef_IsNone(should_be_none)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyErr_Format(PyExc_TypeError, + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXIT_INIT_CHECK); + _PyStackRef should_be_none; + should_be_none = stack_pointer[-1]; + assert(STACK_LEVEL() == 2); + if (!PyStackRef_IsNone(should_be_none)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXTENDED_ARG); - assert(oparg); - opcode = next_instr->op.code; - oparg = oparg << 8 | next_instr->op.arg; - PRE_DISPATCH_GOTO(); - DISPATCH_GOTO(); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXTENDED_ARG); + assert(oparg); + opcode = next_instr->op.code; + oparg = oparg << 8 | next_instr->op.arg; + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); + } + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_SIMPLE); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - /* If value is a unicode object, then we know the result - * of format(value) is value itself. */ - if (!PyUnicode_CheckExact(value_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Format(value_o, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (res_o == NULL) CEVAL_GOTO(pop_1_error); - res = PyStackRef_FromPyObjectSteal(res_o); - } - else { - res = value; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_SIMPLE); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + /* If value is a unicode object, then we know the result + * of format(value) is value itself. */ + if (!PyUnicode_CheckExact(value_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(value_o, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) goto pop_1_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + else { + res = value; + } + stack_pointer[-1] = res; } - stack_pointer[-1] = res; DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_WITH_SPEC); - _PyStackRef value; - _PyStackRef fmt_spec; - _PyStackRef res; - fmt_spec = stack_pointer[-1]; - value = stack_pointer[-2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - PyStackRef_CLOSE(fmt_spec); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_WITH_SPEC); + _PyStackRef value; + _PyStackRef fmt_spec; + _PyStackRef res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + PyStackRef_CLOSE(fmt_spec); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER); - PREDICTED(FOR_ITER); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef iter; - _PyStackRef next; - // _SPECIALIZE_FOR_ITER - { - iter = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ForIter(iter, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(FOR_ITER); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ - } - // _FOR_ITER { - /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_o == NULL) { - if (_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - CEVAL_GOTO(error); - } + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER); + PREDICTED(FOR_ITER); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef iter; + _PyStackRef next; + // _SPECIALIZE_FOR_ITER + { + iter = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); + _Py_Specialize_ForIter(iter, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || + OPCODE_DEFERRED_INC(FOR_ITER); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + // _FOR_ITER + { + /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_o == NULL) { + if (_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!matches) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - /* Jump forward oparg, then skip following END_FOR */ - JUMPBY(oparg + 1); - DISPATCH(); + /* Jump forward oparg, then skip following END_FOR */ + JUMPBY(oparg + 1); + DISPATCH(); + } + next = PyStackRef_FromPyObjectSteal(next_o); + // Common case: no jump, leave it to the code generator } - next = PyStackRef_FromPyObjectSteal(next_o); - // Common case: no jump, leave it to the code generator + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_GEN); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyInterpreterFrame *gen_frame; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); - } - // _FOR_ITER_GEN_FRAME { - iter = stack_pointer[-1]; - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); - STAT_INC(FOR_ITER, hit); - gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, PyStackRef_None); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - gen_frame->previous = frame; - // oparg is the return offset from the next instruction. - frame->return_offset = (uint16_t)( 2 + oparg); - } - // _PUSH_FRAME - { - new_frame = gen_frame; - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_GEN); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyInterpreterFrame *gen_frame; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); + } + // _FOR_ITER_GEN_FRAME + { + iter = stack_pointer[-1]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); + STAT_INC(FOR_ITER, hit); + gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, PyStackRef_None); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + gen_frame->previous = frame; + // oparg is the return offset from the next instruction. + frame->return_offset = (uint16_t)( 2 + oparg); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_LIST); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_LIST { - iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER); - } - // _ITER_JUMP_LIST - { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - STAT_INC(FOR_ITER, hit); - PyListObject *seq = it->it_seq; - if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { - it->it_index = -1; - #ifndef Py_GIL_DISABLED - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_LIST); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_LIST + { + iter = stack_pointer[-1]; + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER); + } + // _ITER_JUMP_LIST + { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); + STAT_INC(FOR_ITER, hit); + PyListObject *seq = it->it_seq; + if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { + it->it_index = -1; + #ifndef Py_GIL_DISABLED + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); + } + #endif + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); } - #endif - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); } + // _ITER_NEXT_LIST + { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); + PyListObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyList_GET_SIZE(seq)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - // _ITER_NEXT_LIST - { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyList_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); - } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_RANGE); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_RANGE - { - iter = stack_pointer[-1]; - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); - } - // _ITER_JUMP_RANGE { - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - STAT_INC(FOR_ITER, hit); - if (r->len <= 0) { - // Jump over END_FOR instruction. - JUMPBY(oparg + 1); - DISPATCH(); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_RANGE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_RANGE + { + iter = stack_pointer[-1]; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); } + // _ITER_JUMP_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + STAT_INC(FOR_ITER, hit); + if (r->len <= 0) { + // Jump over END_FOR instruction. + JUMPBY(oparg + 1); + DISPATCH(); + } + } + // _ITER_NEXT_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + assert(r->len > 0); + long value = r->start; + r->start = value + r->step; + r->len--; + PyObject *res = PyLong_FromLong(value); + if (res == NULL) goto error; + next = PyStackRef_FromPyObjectSteal(res); + } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - // _ITER_NEXT_RANGE - { - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - assert(r->len > 0); - long value = r->start; - r->start = value + r->step; - r->len--; - PyObject *res = PyLong_FromLong(value); - if (res == NULL) CEVAL_GOTO(error); - next = PyStackRef_FromPyObjectSteal(res); - } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_TUPLE); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_TUPLE - { - iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER); - } - // _ITER_JUMP_TUPLE { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - STAT_INC(FOR_ITER, hit); - PyTupleObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_TUPLE + { + iter = stack_pointer[-1]; + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER); + } + // _ITER_JUMP_TUPLE + { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); + STAT_INC(FOR_ITER, hit); + PyTupleObject *seq = it->it_seq; + if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); + } + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); } - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); } + // _ITER_NEXT_TUPLE + { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); + PyTupleObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyTuple_GET_SIZE(seq)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); + } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - // _ITER_NEXT_TUPLE - { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - PyTupleObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); - } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AITER); - _PyStackRef obj; - _PyStackRef iter; - obj = stack_pointer[-1]; - unaryfunc getter = NULL; - PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); - PyObject *iter_o; - PyTypeObject *type = Py_TYPE(obj_o); - if (type->tp_as_async != NULL) { - getter = type->tp_as_async->am_aiter; - } - if (getter == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AITER); + _PyStackRef obj; + _PyStackRef iter; + obj = stack_pointer[-1]; + unaryfunc getter = NULL; + PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); + PyObject *iter_o; + PyTypeObject *type = Py_TYPE(obj_o); + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; + } + if (getter == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(obj); + goto pop_1_error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + iter_o = (*getter)(obj_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); - CEVAL_GOTO(pop_1_error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - iter_o = (*getter)(obj_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(obj); - if (iter_o == NULL) CEVAL_GOTO(pop_1_error); - if (Py_TYPE(iter_o)->tp_as_async == NULL || + if (iter_o == NULL) goto pop_1_error; + if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter_o)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - Py_DECREF(iter_o); - CEVAL_GOTO(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; - DISPATCH(); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(iter_o); + goto error; + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[-1] = iter; + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ANEXT); - _PyStackRef aiter; - _PyStackRef awaitable; - aiter = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (awaitable_o == NULL) { - CEVAL_GOTO(error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ANEXT); + _PyStackRef aiter; + _PyStackRef awaitable; + aiter = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (awaitable_o == NULL) { + goto error; + } + awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); + stack_pointer[0] = awaitable; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); - stack_pointer[0] = awaitable; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AWAITABLE); - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(iterable); - if (iter_o == NULL) CEVAL_GOTO(pop_1_error); - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; - DISPATCH(); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AWAITABLE); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(iterable); + if (iter_o == NULL) goto pop_1_error; + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[-1] = iter; + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ITER); - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - /* before: [obj]; after [getiter(obj)] */ - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(iterable); - if (iter_o == NULL) CEVAL_GOTO(pop_1_error); - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; - DISPATCH(); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ITER); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + /* before: [obj]; after [getiter(obj)] */ + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(iterable); + if (iter_o == NULL) goto pop_1_error; + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[-1] = iter; + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_LEN); - _PyStackRef obj; - _PyStackRef len; - obj = stack_pointer[-1]; - // PUSH(len(TOS)) - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) CEVAL_GOTO(error); - PyObject *len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) CEVAL_GOTO(error); - len = PyStackRef_FromPyObjectSteal(len_o); - stack_pointer[0] = len; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_LEN); + _PyStackRef obj; + _PyStackRef len; + obj = stack_pointer[-1]; + // PUSH(len(TOS)) + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (len_i < 0) goto error; + PyObject *len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) goto error; + len = PyStackRef_FromPyObjectSteal(len_o); + stack_pointer[0] = len; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_YIELD_FROM_ITER); - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - /* before: [obj]; after [getiter(obj)] */ - PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); - if (PyCoro_CheckExact(iterable_o)) { - /* `iterable` is a coroutine */ - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { - /* and it is used in a 'yield from' expression of a - regular generator. */ - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_TypeError, + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_YIELD_FROM_ITER); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + /* before: [obj]; after [getiter(obj)] */ + PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); + if (PyCoro_CheckExact(iterable_o)) { + /* `iterable` is a coroutine */ + if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + /* and it is used in a 'yield from' expression of a + regular generator. */ + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); - } - iter = iterable; - } - else { - if (PyGen_CheckExact(iterable_o)) { + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } iter = iterable; } else { - /* `iterable` is not a generator. */ - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(iterable_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - CEVAL_GOTO(error); + if (PyGen_CheckExact(iterable_o)) { + iter = iterable; + } + else { + /* `iterable` is not a generator. */ + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(iterable_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + goto error; + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + PyStackRef_CLOSE(iterable); } - iter = PyStackRef_FromPyObjectSteal(iter_o); - PyStackRef_CLOSE(iterable); } + stack_pointer[-1] = iter; } - stack_pointer[-1] = iter; DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_FROM); - _PyStackRef from; - _PyStackRef res; - from = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) CEVAL_GOTO(error); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_FROM); + _PyStackRef from; + _PyStackRef res; + from = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) goto error; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_NAME); - _PyStackRef level; - _PyStackRef fromlist; - _PyStackRef res; - fromlist = stack_pointer[-1]; - level = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportName(tstate, frame, name, - PyStackRef_AsPyObjectBorrow(fromlist), - PyStackRef_AsPyObjectBorrow(level)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(level); - PyStackRef_CLOSE(fromlist); - if (res_o == NULL) CEVAL_GOTO(pop_2_error); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_NAME); + _PyStackRef level; + _PyStackRef fromlist; + _PyStackRef res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportName(tstate, frame, name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(level); + PyStackRef_CLOSE(fromlist); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef *func; - _PyStackRef *maybe_self; - _PyStackRef res; - /* Skip 3 cache entries */ - // _MAYBE_EXPAND_METHOD - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - func = &stack_pointer[-2 - oparg]; - maybe_self = &stack_pointer[-1 - oparg]; - if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); - PyStackRef_CLOSE(temp); - } - } - // _MONITOR_CALL { - args = &stack_pointer[-oparg]; - maybe_self = &stack_pointer[-1 - oparg]; - func = &stack_pointer[-2 - oparg]; - int is_meth = !PyStackRef_IsNull(maybe_self[0]); - PyObject *function = PyStackRef_AsPyObjectBorrow(func[0]); - PyObject *arg0; - if (is_meth) { - arg0 = PyStackRef_AsPyObjectBorrow(maybe_self[0]); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef res; + /* Skip 3 cache entries */ + // _MAYBE_EXPAND_METHOD + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + func = &stack_pointer[-2 - oparg]; + maybe_self = &stack_pointer[-1 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); + } } - else { - if (oparg) { - arg0 = PyStackRef_AsPyObjectBorrow(args[0]); + // _MONITOR_CALL + { + args = &stack_pointer[-oparg]; + maybe_self = &stack_pointer[-1 - oparg]; + func = &stack_pointer[-2 - oparg]; + int is_meth = !PyStackRef_IsNull(maybe_self[0]); + PyObject *function = PyStackRef_AsPyObjectBorrow(func[0]); + PyObject *arg0; + if (is_meth) { + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self[0]); } else { - arg0 = &_PyInstrumentation_MISSING; + if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg0 = &_PyInstrumentation_MISSING; + } } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg0 + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) goto error; } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg0 - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) CEVAL_GOTO(error); - } - // _DO_CALL - { - self_or_null = maybe_self; - callable = func; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - // Check if the call can be inlined or not - if (Py_TYPE(callable_o) == &PyFunction_Type && + // _DO_CALL + { + self_or_null = maybe_self; + callable = func; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && tstate->interp->eval_frame == NULL && ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; + } + frame->return_offset = 4 ; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + PyStackRef_CLOSE(callable[0]); + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - args, total_args, NULL, frame - ); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - CEVAL_GOTO(error); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(res_o); + } + } } - frame->return_offset = 4 ; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); for (int i = 0; i < total_args; i++) { PyStackRef_CLOSE(args[i]); } PyStackRef_CLOSE(callable[0]); - { + if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); + int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(res_o); - } + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } - PyStackRef_CLOSE(callable[0]); - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); + GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + } + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - uint32_t version = read_u32(&this_instr[2].cache); - (void)version; - int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2)); - int total_args = oparg + is_meth; - PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3)); - PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING -: PyStackRef_AsPyObjectBorrow(PEEK(total_args + 1)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) CEVAL_GOTO(error); - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(CALL_KW); + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + uint32_t version = read_u32(&this_instr[2].cache); + (void)version; + int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2)); + int total_args = oparg + is_meth; + PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3)); + PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING + : PyStackRef_AsPyObjectBorrow(PEEK(total_args + 1)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) goto error; + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + GO_TO_INSTRUCTION(CALL_KW); + } + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_FOR); - _PyStackRef receiver; - _PyStackRef value; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - /* Need to create a fake StopIteration error here, - * to conform to PEP 380 */ - if (PyStackRef_GenCheck(receiver)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - CEVAL_GOTO(error); + { + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_FOR); + _PyStackRef receiver; + _PyStackRef value; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + /* Need to create a fake StopIteration error here, + * to conform to PEP 380 */ + if (PyStackRef_GenCheck(receiver)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } } + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(value); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_SEND); - _PyStackRef receiver; - _PyStackRef value; - _PyStackRef val; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - CEVAL_GOTO(error); + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_SEND); + _PyStackRef receiver; + _PyStackRef value; + _PyStackRef val; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } } + val = value; + PyStackRef_CLOSE(receiver); + stack_pointer[-2] = val; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - val = value; - PyStackRef_CLOSE(receiver); - stack_pointer[-2] = val; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); - /* Skip 1 cache entry */ - _PyStackRef iter_stackref = TOP(); - PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next != NULL) { - PUSH(PyStackRef_FromPyObjectSteal(next)); - INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); - } - else { - if (_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - CEVAL_GOTO(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); + /* Skip 1 cache entry */ + _PyStackRef iter_stackref = TOP(); + PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next != NULL) { + PUSH(PyStackRef_FromPyObjectSteal(next)); + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || + else { + if (_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!matches) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - /* Skip END_FOR */ - JUMPBY(oparg + 1); + /* Skip END_FOR */ + JUMPBY(oparg + 1); + } } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); - _PyFrame_SetStackPointer(frame, stack_pointer); - int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_opcode < 0) CEVAL_GOTO(error); - next_instr = this_instr; - if (_PyOpcode_Caches[next_opcode]) { - PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + _PyFrame_SetStackPointer(frame, stack_pointer); + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_opcode < 0) goto error; + next_instr = this_instr; + if (_PyOpcode_Caches[next_opcode]) { + PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); + } + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); } - assert(next_opcode > 0 && next_opcode < 256); - opcode = next_opcode; - DISPATCH_GOTO(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); - /* Skip 1 cache entry */ - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - } - } - // _MONITOR_JUMP_BACKWARD { - INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + } + } + // _MONITOR_JUMP_BACKWARD + { + INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); + } } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); - DISPATCH(); + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_LINE); - int original_opcode = 0; - if (tstate->tracing) { - PyCodeObject *code = _PyFrame_GetCode(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - original_opcode = code->_co_monitoring->lines[(int)(this_instr - _PyFrame_GetBytecode(frame))].original_opcode; - stack_pointer = _PyFrame_GetStackPointer(frame); - next_instr = this_instr; - } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - original_opcode = _Py_call_instrumentation_line( - tstate, frame, this_instr, prev_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (original_opcode < 0) { - next_instr = this_instr+1; - CEVAL_GOTO(error); + { + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_LINE); + int original_opcode = 0; + if (tstate->tracing) { + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = code->_co_monitoring->lines[(int)(this_instr - _PyFrame_GetBytecode(frame))].original_opcode; + stack_pointer = _PyFrame_GetStackPointer(frame); + next_instr = this_instr; + } else { + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = _Py_call_instrumentation_line( + tstate, frame, this_instr, prev_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = this_instr+1; + goto error; + } + next_instr = frame->instr_ptr; + if (next_instr != this_instr) { + DISPATCH(); + } } - next_instr = frame->instr_ptr; - if (next_instr != this_instr) { - DISPATCH(); + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + /* Prevent the underlying instruction from specializing + * and overwriting the instrumentation. */ + PAUSE_ADAPTIVE_COUNTER(cache->counter); } + opcode = original_opcode; + DISPATCH_GOTO(); } - if (_PyOpcode_Caches[original_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - /* Prevent the underlying instruction from specializing - * and overwriting the instrumentation. */ - PAUSE_ADAPTIVE_COUNTER(cache->counter); - } - opcode = original_opcode; - DISPATCH_GOTO(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); - /* Skip 1 cache entry */ - // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we - // don't want to specialize instrumented instructions - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); + /* Skip 1 cache entry */ + // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we + // don't want to specialize instrumented instructions + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + } + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); - (void)this_instr; // INSTRUMENTED_JUMP requires this_instr - INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); - DISPATCH(); + { + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); + (void)this_instr; // INSTRUMENTED_JUMP requires this_instr + INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); - _PyStackRef iter; - iter = stack_pointer[-1]; - INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); - PyStackRef_CLOSE(iter); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + { + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); + _PyStackRef iter; + iter = stack_pointer[-1]; + INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); + PyStackRef_CLOSE(iter); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); - /* Skip 1 cache entry */ - _PyStackRef cond = POP(); - assert(PyStackRef_BoolCheck(cond)); - int jump = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - DISPATCH(); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS){ + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); + /* Skip 1 cache entry */ + _PyStackRef cond = POP(); + assert(PyStackRef_BoolCheck(cond)); + int jump = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); - /* Skip 1 cache entry */ - _PyStackRef value_stackref = POP(); - int jump = PyStackRef_IsNone(value_stackref); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - else { - PyStackRef_CLOSE(value_stackref); + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); + /* Skip 1 cache entry */ + _PyStackRef value_stackref = POP(); + int jump = PyStackRef_IsNone(value_stackref); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + else { + PyStackRef_CLOSE(value_stackref); + } } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); - /* Skip 1 cache entry */ - _PyStackRef value_stackref = POP(); - int jump = !PyStackRef_IsNone(value_stackref); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - PyStackRef_CLOSE(value_stackref); - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - DISPATCH(); + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); + /* Skip 1 cache entry */ + _PyStackRef value_stackref = POP(); + int jump = !PyStackRef_IsNone(value_stackref); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + PyStackRef_CLOSE(value_stackref); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); - /* Skip 1 cache entry */ - _PyStackRef cond = POP(); - assert(PyStackRef_BoolCheck(cond)); - int jump = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - DISPATCH(); + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); + /* Skip 1 cache entry */ + _PyStackRef cond = POP(); + assert(PyStackRef_BoolCheck(cond)); + int jump = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RESUME); - // _LOAD_BYTECODE { - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RESUME); + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) CEVAL_GOTO(error); - _PyFrame_SetStackPointer(frame, stack_pointer); - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; - frame->instr_ptr = bytecode + off; - // Make sure this_instr gets reset correctley for any uops that - // follow - next_instr = frame->instr_ptr; - DISPATCH(); - } - #endif - } - // _MAYBE_INSTRUMENT - { - if (tstate->tracing == 0) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - CEVAL_GOTO(error); - } - next_instr = this_instr; + if (bytecode == NULL) goto error; + _PyFrame_SetStackPointer(frame, stack_pointer); + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + // Make sure this_instr gets reset correctley for any uops that + // follow + next_instr = frame->instr_ptr; DISPATCH(); } + #endif } - } - // _CHECK_PERIODIC_IF_NOT_YIELD_FROM - { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); + // _MAYBE_INSTRUMENT + { + if (tstate->tracing == 0) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } + next_instr = this_instr; + DISPATCH(); + } } } - } - // _MONITOR_RESUME - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) CEVAL_GOTO(error); - if (frame->instr_ptr != this_instr) { - /* Instrumentation has jumped */ - next_instr = frame->instr_ptr; + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + } + } + } + // _MONITOR_RESUME + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation( + tstate, oparg > 0, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) goto error; + if (frame->instr_ptr != this_instr) { + /* Instrumentation has jumped */ + next_instr = frame->instr_ptr; + } } } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); - _PyStackRef val; - _PyStackRef retval; - _PyStackRef res; - // _RETURN_VALUE_EVENT - { - val = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) CEVAL_GOTO(error); - } - // _RETURN_VALUE { - retval = val; - #if TIER_ONE - assert(!frame->is_entry_frame); - #endif - _PyStackRef temp = retval; - stack_pointer += -1; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); + _PyStackRef val; + _PyStackRef retval; + _PyStackRef res; + // _RETURN_VALUE_EVENT + { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) goto error; + } + // _RETURN_VALUE + { + retval = val; + #if TIER_ONE + assert(!frame->is_entry_frame); + #endif + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(frame->return_offset); + res = temp; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(EMPTY()); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(frame->return_offset); - res = temp; - LLTRACE_RESUME_FRAME(); } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); - _PyStackRef val; - _PyStackRef retval; - _PyStackRef value; - // _YIELD_VALUE_EVENT { - val = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - CEVAL_GOTO(error); - } - if (frame->instr_ptr != this_instr) { - next_instr = frame->instr_ptr; - DISPATCH(); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + _PyStackRef val; + _PyStackRef retval; + _PyStackRef value; + // _YIELD_VALUE_EVENT + { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } + if (frame->instr_ptr != this_instr) { + next_instr = frame->instr_ptr; + DISPATCH(); + } } - } - // _YIELD_VALUE - { - retval = val; - // NOTE: It's important that YIELD_VALUE never raises an exception! - // The compiler treats any exception raised here as a failed close() - // or throw() call. - #if TIER_ONE - assert(!frame->is_entry_frame); - #endif - frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + // _YIELD_VALUE + { + retval = val; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + #if TIER_ONE + assert(!frame->is_entry_frame); + #endif + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - value = temp; - LLTRACE_RESUME_FRAME(); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + value = temp; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INTERPRETER_EXIT); - _PyStackRef retval; - retval = stack_pointer[-1]; - assert(frame->is_entry_frame); - assert(_PyFrame_IsIncomplete(frame)); - /* Restore previous frame and return. */ - tstate->current_frame = frame->previous; - assert(!_PyErr_Occurred(tstate)); - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - PyObject *result = PyStackRef_AsPyObjectSteal(retval); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - /* Not strictly necessary, but prevents warnings */ - return result; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INTERPRETER_EXIT); + _PyStackRef retval; + retval = stack_pointer[-1]; + assert(frame->is_entry_frame); + assert(_PyFrame_IsIncomplete(frame)); + /* Restore previous frame and return. */ + tstate->current_frame = frame->previous; + assert(!_PyErr_Occurred(tstate)); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + PyObject *result = PyStackRef_AsPyObjectSteal(retval); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + /* Not strictly necessary, but prevents warnings */ + return result; + } + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IS_OP); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IS_OP); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD); - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); - } - } - // _JUMP_BACKWARD { - uint16_t the_counter = read_u16(&this_instr[1].cache); - (void)the_counter; - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); - #ifdef _Py_TIER2 - #if ENABLE_SPECIALIZATION - _Py_BackoffCounter counter = this_instr[1].counter; - if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) { - _Py_CODEUNIT *start = this_instr; - /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ - while (oparg > 255) { - oparg >>= 8; - start--; - } - _PyExecutorObject *executor; - _PyFrame_SetStackPointer(frame, stack_pointer); - int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (optimized <= 0) { - this_instr[1].counter = restart_backoff_counter(counter); - if (optimized < 0) CEVAL_GOTO(error); - } - else { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD); + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { _PyFrame_SetStackPointer(frame, stack_pointer); - this_instr[1].counter = initial_jump_backoff_counter(); + int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - assert(tstate->previous_executor == NULL); - tstate->previous_executor = Py_None; - GOTO_TIER_TWO(executor); + if (err != 0) goto error; } } - else { - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + // _JUMP_BACKWARD + { + uint16_t the_counter = read_u16(&this_instr[1].cache); + (void)the_counter; + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + #ifdef _Py_TIER2 + #if ENABLE_SPECIALIZATION + _Py_BackoffCounter counter = this_instr[1].counter; + if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) { + _Py_CODEUNIT *start = this_instr; + /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ + while (oparg > 255) { + oparg >>= 8; + start--; + } + _PyExecutorObject *executor; + _PyFrame_SetStackPointer(frame, stack_pointer); + int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (optimized <= 0) { + this_instr[1].counter = restart_backoff_counter(counter); + if (optimized < 0) goto error; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + this_instr[1].counter = initial_jump_backoff_counter(); + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(tstate->previous_executor == NULL); + tstate->previous_executor = Py_None; + GOTO_TIER_TWO(executor); + } + } + else { + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + #endif /* ENABLE_SPECIALIZATION */ + #endif /* _Py_TIER2 */ } - #endif /* ENABLE_SPECIALIZATION */ - #endif /* _Py_TIER2 */ } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); - /* This bytecode is used in the `yield from` or `await` loop. - * If there is an interrupt, we want it handled in the innermost - * generator or coroutine, so we deliberately do not check it here. - * (see bpo-30039). - */ - JUMPBY(-oparg); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + JUMPBY(-oparg); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_FORWARD); - JUMPBY(oparg); - DISPATCH(); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_FORWARD); + JUMPBY(oparg); + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_APPEND); - _PyStackRef list; - _PyStackRef v; - v = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), - PyStackRef_AsPyObjectSteal(v)); - if (err < 0) CEVAL_GOTO(pop_1_error); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_APPEND); + _PyStackRef list; + _PyStackRef v; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)); + if (err < 0) goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_EXTEND); - _PyStackRef list_st; - _PyStackRef iterable_st; - iterable_st = stack_pointer[-1]; - list_st = stack_pointer[-2 - (oparg-1)]; - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (none_val == NULL) { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_EXTEND); + _PyStackRef list_st; + _PyStackRef iterable_st; + iterable_st = stack_pointer[-1]; + list_st = stack_pointer[-2 - (oparg-1)]; + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) - { + if (none_val == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Clear(tstate); - _PyErr_Format(tstate, PyExc_TypeError, + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + PyStackRef_CLOSE(iterable_st); + goto pop_1_error; } + assert(Py_IsNone(none_val)); PyStackRef_CLOSE(iterable_st); - CEVAL_GOTO(pop_1_error); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - assert(Py_IsNone(none_val)); - PyStackRef_CLOSE(iterable_st); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR); - PREDICTED(LOAD_ATTR); - _Py_CODEUNIT* const this_instr = next_instr - 10; - (void)this_instr; - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self_or_null = PyStackRef_NULL; - // _SPECIALIZE_LOAD_ATTR { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR); + PREDICTED(LOAD_ATTR); + _Py_CODEUNIT* const this_instr = next_instr - 10; + (void)this_instr; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self_or_null = PyStackRef_NULL; + // _SPECIALIZE_LOAD_ATTR + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } - OPCODE_DEFERRED_INC(LOAD_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 8 cache entries */ - // _LOAD_ATTR - { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - PyObject *attr_o; - if (oparg & 1) { - /* Designed to work in tandem with CALL, pushes two values. */ - attr_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (is_meth) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - meth | self | arg1 | ... | argN - */ - assert(attr_o != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership + /* Skip 8 cache entries */ + // _LOAD_ATTR + { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + PyObject *attr_o; + if (oparg & 1) { + /* Designed to work in tandem with CALL, pushes two values. */ + attr_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (is_meth) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr_o != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + } + else { + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + meth | NULL | arg1 | ... | argN + */ + PyStackRef_CLOSE(owner); + if (attr_o == NULL) goto pop_1_error; + self_or_null = PyStackRef_NULL; + } } else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - meth | NULL | arg1 | ... | argN - */ - PyStackRef_CLOSE(owner); - if (attr_o == NULL) CEVAL_GOTO(pop_1_error); - self_or_null = PyStackRef_NULL; - } - } - else { - /* Classic, pushes one value. */ - _PyFrame_SetStackPointer(frame, stack_pointer); - attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(owner); - if (attr_o == NULL) CEVAL_GOTO(pop_1_error); - /* We need to define self_or_null on all paths */ - self_or_null = PyStackRef_NULL; + /* Classic, pushes one value. */ + _PyFrame_SetStackPointer(frame, stack_pointer); + attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(owner); + if (attr_o == NULL) goto pop_1_error; + /* We need to define self_or_null on all paths */ + self_or_null = PyStackRef_NULL; + } + attr = PyStackRef_FromPyObjectSteal(attr_o); } - attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = self_or_null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _CHECK_ATTR_CLASS - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); - assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_CLASS { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = PyStackRef_FromPyObjectNew(descr); - null = PyStackRef_NULL; - PyStackRef_CLOSE(owner); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + assert(type_version != 0); + DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _CHECK_ATTR_CLASS - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); - assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); - } - // _GUARD_TYPE_VERSION { - uint32_t type_version = read_u32(&this_instr[4].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - // _LOAD_ATTR_CLASS - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = PyStackRef_FromPyObjectNew(descr); - null = PyStackRef_NULL; - PyStackRef_CLOSE(owner); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + assert(type_version != 0); + DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); + } + // _GUARD_TYPE_VERSION + { + uint32_t type_version = read_u32(&this_instr[4].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - /* Skip 1 cache entry */ - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *getattribute = read_obj(&this_instr[6].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner_o); - assert(type_version != 0); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); - assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)getattribute; - assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( - tstate, PyStackRef_FromPyObjectNew(f), 2, frame); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); - new_frame->localsplus[0] = owner; - new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); - frame->return_offset = 10 ; - DISPATCH_INLINED(new_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + /* Skip 1 cache entry */ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - // _CHECK_MANAGED_OBJECT_HAS_VALUES - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); - } - // _LOAD_ATTR_INSTANCE_VALUE - { - uint16_t offset = read_u16(&this_instr[4].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *getattribute = read_obj(&this_instr[6].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = *value_ptr; - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + assert((oparg & 1) == 0); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + PyTypeObject *cls = Py_TYPE(owner_o); + assert(type_version != 0); + DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)getattribute; + assert(func_version != 0); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 2); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); - null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectSteal(attr_o); - PyStackRef_CLOSE(owner); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( + tstate, PyStackRef_FromPyObjectNew(f), 2, frame); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). + STACK_SHRINK(1); + new_frame->localsplus[0] = owner; + new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); + frame->return_offset = 10 ; + DISPATCH_INLINED(new_frame); + } + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS){ + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _CHECK_MANAGED_OBJECT_HAS_VALUES + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + } + // _LOAD_ATTR_INSTANCE_VALUE + { + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = *value_ptr; + DEOPT_IF(attr_o == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr_o); + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectSteal(attr_o); + PyStackRef_CLOSE(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - // _CHECK_ATTR_METHOD_LAZY_DICT - { - uint16_t dictoffset = read_u16(&this_instr[4].cache); - char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = *(PyObject **)ptr; - /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR); - } - /* Skip 1 cache entry */ - // _LOAD_ATTR_METHOD_LAZY_DICT { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _CHECK_ATTR_METHOD_LAZY_DICT + { + uint16_t dictoffset = read_u16(&this_instr[4].cache); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = *(PyObject **)ptr; + /* This object has a __dict__, just not yet created */ + DEOPT_IF(dict != NULL, LOAD_ATTR); + } + /* Skip 1 cache entry */ + // _LOAD_ATTR_METHOD_LAZY_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_METHOD_NO_DICT { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); - } - // _LOAD_ATTR_METHOD_WITH_VALUES { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - /* Cached method object */ - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + } + // _LOAD_ATTR_METHOD_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + /* Cached method object */ + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_MODULE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - PyDictKeysObject *mod_keys; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _CHECK_ATTR_MODULE_PUSH_KEYS { - owner = stack_pointer[-1]; - uint32_t dict_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; - assert(dict != NULL); - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR); - mod_keys = keys; - } - // _LOAD_ATTR_MODULE_FROM_KEYS - { - uint16_t index = read_u16(&this_instr[4].cache); - assert(mod_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries)); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); - // Clear mod_keys from stack in case we need to deopt - DEOPT_IF(attr_o == NULL, LOAD_ATTR); - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); - if (!increfed) { - DEOPT_IF(true, LOAD_ATTR); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + PyDictKeysObject *mod_keys; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_MODULE_PUSH_KEYS + { + owner = stack_pointer[-1]; + uint32_t dict_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; + assert(dict != NULL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR); + mod_keys = keys; + } + // _LOAD_ATTR_MODULE_FROM_KEYS + { + uint16_t index = read_u16(&this_instr[4].cache); + assert(mod_keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries)); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); + // Clear mod_keys from stack in case we need to deopt + DEOPT_IF(attr_o == NULL, LOAD_ATTR); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + DEOPT_IF(true, LOAD_ATTR); + } + #else + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); } - #else - Py_INCREF(attr_o); - attr = PyStackRef_FromPyObjectSteal(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; - PyStackRef_CLOSE(owner); + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT { - PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - PyStackRef_CLOSE(owner); - attr = PyStackRef_FromPyObjectNew(descr); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + PyStackRef_CLOSE(owner); + attr = PyStackRef_FromPyObjectNew(descr); + } + stack_pointer[-1] = attr; } - stack_pointer[-1] = attr; DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); - } - // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { - PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - PyStackRef_CLOSE(owner); - attr = PyStackRef_FromPyObjectNew(descr); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + } + // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + PyStackRef_CLOSE(owner); + attr = PyStackRef_FromPyObjectNew(descr); + } + stack_pointer[-1] = attr; } - stack_pointer[-1] = attr; DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - } - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_PROPERTY_FRAME - { - PyObject *fget = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - assert(Py_IS_TYPE(fget, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)fget; - PyCodeObject *code = (PyCodeObject *)f->func_code; - DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR); - DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR); - DEOPT_IF(code->co_argcount != 1, LOAD_ATTR); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); - new_frame->localsplus[0] = owner; - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - DISPATCH(); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + } + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_PROPERTY_FRAME + { + PyObject *fget = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_IS_TYPE(fget, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)fget; + PyCodeObject *code = (PyCodeObject *)f->func_code; + DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR); + DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR); + DEOPT_IF(code->co_argcount != 1, LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); + new_frame->localsplus[0] = owner; + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_SLOT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - // _LOAD_ATTR_SLOT { - uint16_t index = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - char *addr = (char *)owner_o + index; - PyObject *attr_o = *(PyObject **)addr; - DEOPT_IF(attr_o == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectNew(attr_o); - PyStackRef_CLOSE(owner); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _LOAD_ATTR_SLOT + { + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + char *addr = (char *)owner_o + index; + PyObject *attr_o = *(PyObject **)addr; + DEOPT_IF(attr_o == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectNew(attr_o); + PyStackRef_CLOSE(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - // _CHECK_ATTR_WITH_HINT { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, LOAD_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); - } - // _LOAD_ATTR_WITH_HINT - { - uint16_t hint = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject *attr_o; - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), LOAD_ATTR); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); - attr_o = ep->me_value; - DEOPT_IF(attr_o == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); - attr = PyStackRef_FromPyObjectSteal(attr_o); - null = PyStackRef_NULL; - PyStackRef_CLOSE(owner); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _CHECK_ATTR_WITH_HINT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + DEOPT_IF(dict == NULL, LOAD_ATTR); + assert(PyDict_CheckExact((PyObject *)dict)); + } + // _LOAD_ATTR_WITH_HINT + { + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject *attr_o; + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), LOAD_ATTR); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, LOAD_ATTR); + attr_o = ep->me_value; + DEOPT_IF(attr_o == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_BUILD_CLASS); - _PyStackRef bc; - PyObject *bc_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) CEVAL_GOTO(error); - if (bc_o == NULL) { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_BUILD_CLASS); + _PyStackRef bc; + PyObject *bc_o; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_NameError, - "__build_class__ not found"); + int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + if (err < 0) goto error; + if (bc_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + bc = PyStackRef_FromPyObjectSteal(bc_o); + stack_pointer[0] = bc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - bc = PyStackRef_FromPyObjectSteal(bc_o); - stack_pointer[0] = bc; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); - _PyStackRef value; - // Keep in sync with _common_constants in opcode.py - // If we ever have more than two constants, use a lookup table - PyObject *val; - if (oparg == CONSTANT_ASSERTIONERROR) { - val = PyExc_AssertionError; - } - else { - assert(oparg == CONSTANT_NOTIMPLEMENTEDERROR); - val = PyExc_NotImplementedError; + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); + _PyStackRef value; + // Keep in sync with _common_constants in opcode.py + // If we ever have more than two constants, use a lookup table + PyObject *val; + if (oparg == CONSTANT_ASSERTIONERROR) { + val = PyExc_AssertionError; + } + else { + assert(oparg == CONSTANT_NOTIMPLEMENTEDERROR); + val = PyExc_NotImplementedError; + } + value = PyStackRef_FromPyObjectImmortal(val); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - value = PyStackRef_FromPyObjectImmortal(val); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST); - PREDICTED(LOAD_CONST); - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; - _PyStackRef value; - /* We can't do this in the bytecode compiler as - * marshalling can intern strings and make them immortal. */ - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectNew(obj); - #if ENABLE_SPECIALIZATION - if (this_instr->op.code == LOAD_CONST) { - this_instr->op.code = _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL; - } - #endif - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST); + PREDICTED(LOAD_CONST); + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; + _PyStackRef value; + /* We can't do this in the bytecode compiler as + * marshalling can intern strings and make them immortal. */ + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectNew(obj); + #if ENABLE_SPECIALIZATION + if (this_instr->op.code == LOAD_CONST) { + this_instr->op.code = _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL; + } + #endif + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST_IMMORTAL); - static_assert(0 == 0, "incorrect cache size"); - _PyStackRef value; - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - assert(_Py_IsImmortal(obj)); - value = PyStackRef_FromPyObjectImmortal(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST_IMMORTAL); + static_assert(0 == 0, "incorrect cache size"); + _PyStackRef value; + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + assert(_Py_IsImmortal(obj)); + value = PyStackRef_FromPyObjectImmortal(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_MORTAL(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST_MORTAL); - static_assert(0 == 0, "incorrect cache size"); - _PyStackRef value; - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectNew(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST_MORTAL); + static_assert(0 == 0, "incorrect cache size"); + _PyStackRef value; + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectNew(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_DEREF); - _PyStackRef value; - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *value_o = PyCell_GetRef(cell); - if (value_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_DEREF); + _PyStackRef value; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *value_o = PyCell_GetRef(cell); + if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + value = PyStackRef_FromPyObjectSteal(value_o); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - value = PyStackRef_FromPyObjectSteal(value_o); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST); - _PyStackRef value; - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST); + _PyStackRef value; + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); - _PyStackRef value; - value = GETLOCAL(oparg); - // do not use SETLOCAL here, it decrefs the old value - GETLOCAL(oparg) = PyStackRef_NULL; - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); + _PyStackRef value; + value = GETLOCAL(oparg); + // do not use SETLOCAL here, it decrefs the old value + GETLOCAL(oparg) = PyStackRef_NULL; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_CHECK); - _PyStackRef value; - _PyStackRef value_s = GETLOCAL(oparg); - if (PyStackRef_IsNull(value_s)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_CHECK); + _PyStackRef value; + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + value = PyStackRef_DUP(value_s); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - value = PyStackRef_DUP(value_s); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); - _PyStackRef value1; - _PyStackRef value2; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - value1 = PyStackRef_DUP(GETLOCAL(oparg1)); - value2 = PyStackRef_DUP(GETLOCAL(oparg2)); - stack_pointer[0] = value1; - stack_pointer[1] = value2; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); + _PyStackRef value1; + _PyStackRef value2; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = PyStackRef_DUP(GETLOCAL(oparg1)); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); - _PyStackRef class_dict_st; - _PyStackRef value; - class_dict_st = stack_pointer[-1]; - PyObject *value_o; - PyObject *name; - PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); - assert(class_dict); - assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); - name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - CEVAL_GOTO(error); - } - if (!value_o) { - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - value_o = PyCell_GetRef(cell); - if (value_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); + _PyStackRef class_dict_st; + _PyStackRef value; + class_dict_st = stack_pointer[-1]; + PyObject *value_o; + PyObject *name; + PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); + assert(class_dict); + assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + goto error; } - } - PyStackRef_CLOSE(class_dict_st); - value = PyStackRef_FromPyObjectSteal(value_o); - stack_pointer[-1] = value; - DISPATCH(); + if (!value_o) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + value_o = PyCell_GetRef(cell); + if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + } + PyStackRef_CLOSE(class_dict_st); + value = PyStackRef_FromPyObjectSteal(value_o); + stack_pointer[-1] = value; + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); - _PyStackRef mod_or_class_dict; - _PyStackRef v; - mod_or_class_dict = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *v_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(mod_or_class_dict); - if (err < 0) CEVAL_GOTO(pop_1_error); - if (v_o == NULL) { - if (PyDict_CheckExact(GLOBALS()) + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); + _PyStackRef mod_or_class_dict; + _PyStackRef v; + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *v_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(mod_or_class_dict); + if (err < 0) goto pop_1_error; + if (v_o == NULL) { + if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) - { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + goto error; } - CEVAL_GOTO(error); } - } - else { - /* Slow-path if globals or builtins is not a dict */ - /* namespace 1: globals */ - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) CEVAL_GOTO(error); - if (v_o == NULL) { - /* namespace 2: builtins */ + else { + /* Slow-path if globals or builtins is not a dict */ + /* namespace 1: globals */ + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); + int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) CEVAL_GOTO(error); + if (err < 0) goto error; if (v_o == NULL) { + /* namespace 2: builtins */ _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + if (err < 0) goto error; + if (v_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } } } + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[-1] = v; } - v = PyStackRef_FromPyObjectSteal(v_o); - stack_pointer[-1] = v; DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL); - PREDICTED(LOAD_GLOBAL); - _Py_CODEUNIT* const this_instr = next_instr - 5; - (void)this_instr; - _PyStackRef *res; - _PyStackRef null = PyStackRef_NULL; - // _SPECIALIZE_LOAD_GLOBAL { - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL); + PREDICTED(LOAD_GLOBAL); + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + _PyStackRef *res; + _PyStackRef null = PyStackRef_NULL; + // _SPECIALIZE_LOAD_GLOBAL + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_GLOBAL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + // _LOAD_GLOBAL + { + res = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + if (PyStackRef_IsNull(*res)) goto error; + null = PyStackRef_NULL; } - OPCODE_DEFERRED_INC(LOAD_GLOBAL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 1 cache entry */ - /* Skip 1 cache entry */ - /* Skip 1 cache entry */ - // _LOAD_GLOBAL - { - res = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*res)) CEVAL_GOTO(error); - null = PyStackRef_NULL; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - PyDictKeysObject *builtins_keys; - _PyStackRef res; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_GLOBALS_VERSION - { - uint16_t version = read_u16(&this_instr[2].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(keys)); - } - // _GUARD_BUILTINS_VERSION_PUSH_KEYS - { - uint16_t version = read_u16(&this_instr[3].cache); - PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); - builtins_keys = keys; - assert(DK_IS_UNICODE(builtins_keys)); - } - // _LOAD_GLOBAL_BUILTINS_FROM_KEYS - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL); - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL); - #else - Py_INCREF(res_o); - res = PyStackRef_FromPyObjectSteal(res_o); - #endif - STAT_INC(LOAD_GLOBAL, hit); - null = PyStackRef_NULL; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + PyDictKeysObject *builtins_keys; + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION + { + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(keys)); + } + // _GUARD_BUILTINS_VERSION_PUSH_KEYS + { + uint16_t version = read_u16(&this_instr[3].cache); + PyDictObject *dict = (PyDictObject *)BUILTINS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + builtins_keys = keys; + assert(DK_IS_UNICODE(builtins_keys)); + } + // _LOAD_GLOBAL_BUILTINS_FROM_KEYS + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + DEOPT_IF(!increfed, LOAD_GLOBAL); + #else + Py_INCREF(res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + null = PyStackRef_NULL; + } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - PyDictKeysObject *globals_keys; - _PyStackRef res; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_GLOBALS_VERSION_PUSH_KEYS - { - uint16_t version = read_u16(&this_instr[2].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); - globals_keys = keys; - assert(DK_IS_UNICODE(globals_keys)); - } - /* Skip 1 cache entry */ - // _LOAD_GLOBAL_MODULE_FROM_KEYS - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL); - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL); - #else - Py_INCREF(res_o); - res = PyStackRef_FromPyObjectSteal(res_o); - #endif - STAT_INC(LOAD_GLOBAL, hit); - null = PyStackRef_NULL; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + PyDictKeysObject *globals_keys; + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION_PUSH_KEYS + { + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + globals_keys = keys; + assert(DK_IS_UNICODE(globals_keys)); + } + /* Skip 1 cache entry */ + // _LOAD_GLOBAL_MODULE_FROM_KEYS + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + DEOPT_IF(!increfed, LOAD_GLOBAL); + #else + Py_INCREF(res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + null = PyStackRef_NULL; + } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_LOCALS); - _PyStackRef locals; - PyObject *l = LOCALS(); - if (l == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_SystemError, + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_LOCALS); + _PyStackRef locals; + PyObject *l = LOCALS(); + if (l == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + locals = PyStackRef_FromPyObjectNew(l); + stack_pointer[0] = locals; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - locals = PyStackRef_FromPyObjectNew(l); - stack_pointer[0] = locals; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_NAME); - _PyStackRef v; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *v_o = _PyEval_LoadName(tstate, frame, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) CEVAL_GOTO(error); - v = PyStackRef_FromPyObjectSteal(v_o); - stack_pointer[0] = v; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_NAME); + _PyStackRef v; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) goto error; + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_SMALL_INT); - _PyStackRef value; - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectImmortal(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SMALL_INT); + _PyStackRef value; + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectImmortal(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_SPECIAL); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self_or_null; - owner = stack_pointer[-1]; - assert(oparg <= SPECIAL_MAX); - PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); - PyObject *name = _Py_SpecialMethods[oparg].name; - PyObject *self_or_null_o; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = _PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SPECIAL); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self_or_null; + owner = stack_pointer[-1]; + assert(oparg <= SPECIAL_MAX); + PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); + PyObject *name = _Py_SpecialMethods[oparg].name; + PyObject *self_or_null_o; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = _PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, _Py_SpecialMethods[oparg].error, Py_TYPE(owner_o)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + goto error; } - CEVAL_GOTO(error); + attr = PyStackRef_FromPyObjectSteal(attr_o); + self_or_null = self_or_null_o == NULL ? + PyStackRef_NULL : PyStackRef_FromPyObjectSteal(self_or_null_o); + stack_pointer[0] = attr; + stack_pointer[1] = self_or_null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } - attr = PyStackRef_FromPyObjectSteal(attr_o); - self_or_null = self_or_null_o == NULL ? - PyStackRef_NULL : PyStackRef_FromPyObjectSteal(self_or_null_o); - stack_pointer[0] = attr; - stack_pointer[1] = self_or_null; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR); - PREDICTED(LOAD_SUPER_ATTR); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - // _SPECIALIZE_LOAD_SUPER_ATTR - { - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - int load_method = oparg & 1; - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _LOAD_SUPER_ATTR { - self_st = stack_pointer[-1]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - CEVAL_GOTO(pop_3_error); - } - } - // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = {class, self}; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR); + PREDICTED(LOAD_SUPER_ATTR); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + // _SPECIALIZE_LOAD_SUPER_ATTR + { + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + int load_method = oparg & 1; + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); } - else { + OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _LOAD_SUPER_ATTR + { + self_st = stack_pointer[-1]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, + tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, global_super, arg); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(super); + if (err) { + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + goto pop_3_error; } } - } - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - if (super == NULL) CEVAL_GOTO(pop_3_error); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - stack_pointer += -3; + // we make no attempt to optimize here; specializations should + // handle any case whose performance we care about + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(super); + } + } + } + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + if (super == NULL) goto pop_3_error; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(super); + if (attr_o == NULL) goto error; + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; + } + stack_pointer[0] = attr; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(super, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - Py_DECREF(super); - if (attr_o == NULL) CEVAL_GOTO(error); - attr = PyStackRef_FromPyObjectSteal(attr_o); - null = PyStackRef_NULL; } - stack_pointer[0] = attr; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr_st; - /* Skip 1 cache entry */ - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - if (attr == NULL) CEVAL_GOTO(pop_3_error); - attr_st = PyStackRef_FromPyObjectSteal(attr); - stack_pointer[-3] = attr_st; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr_st; + /* Skip 1 cache entry */ + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(!(oparg & 1)); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + if (attr == NULL) goto pop_3_error; + attr_st = PyStackRef_FromPyObjectSteal(attr); + stack_pointer[-3] = attr_st; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef self_or_null; - /* Skip 1 cache entry */ - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - PyTypeObject *cls = (PyTypeObject *)class; - int method_found = 0; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - CEVAL_GOTO(error); - } - if (method_found) { - self_or_null = self_st; // transfer ownership - } else { - PyStackRef_CLOSE(self_st); - self_or_null = PyStackRef_NULL; + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef self_or_null; + /* Skip 1 cache entry */ + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(oparg & 1); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyTypeObject *cls = (PyTypeObject *)class; + int method_found = 0; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + goto error; + } + if (method_found) { + self_or_null = self_st; // transfer ownership + } else { + PyStackRef_CLOSE(self_st); + self_or_null = PyStackRef_NULL; + } + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(global_super_st); + attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(global_super_st); - attr = PyStackRef_FromPyObjectSteal(attr_o); - stack_pointer[-3] = attr; - stack_pointer[-2] = self_or_null; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_CELL); - // "initial" is probably NULL but not if it's an arg (or set - // via the f_locals proxy before MAKE_CELL has run). - PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *cell = PyCell_New(initial); - if (cell == NULL) { - CEVAL_GOTO(error); - } - SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell)); - DISPATCH(); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_CELL); + // "initial" is probably NULL but not if it's an arg (or set + // via the f_locals proxy before MAKE_CELL has run). + PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *cell = PyCell_New(initial); + if (cell == NULL) { + goto error; + } + SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell)); + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_FUNCTION); - _PyStackRef codeobj_st; - _PyStackRef func; - codeobj_st = stack_pointer[-1]; - PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(codeobj_st); - if (func_obj == NULL) CEVAL_GOTO(pop_1_error); - _PyFunction_SetVersion( + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_FUNCTION); + _PyStackRef codeobj_st; + _PyStackRef func; + codeobj_st = stack_pointer[-1]; + PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(codeobj_st); + if (func_obj == NULL) goto pop_1_error; + _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); - func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); - stack_pointer[-1] = func; - DISPATCH(); + func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); + stack_pointer[-1] = func; + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAP_ADD); - _PyStackRef dict_st; - _PyStackRef key; - _PyStackRef value; - value = stack_pointer[-1]; - key = stack_pointer[-2]; - dict_st = stack_pointer[-3 - (oparg - 1)]; - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); - /* dict[key] = value */ - // Do not DECREF INPUTS because the function steals the references - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2( - (PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(key), - PyStackRef_AsPyObjectSteal(value) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(pop_2_error); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAP_ADD); + _PyStackRef dict_st; + _PyStackRef key; + _PyStackRef value; + value = stack_pointer[-1]; + key = stack_pointer[-2]; + dict_st = stack_pointer[-3 - (oparg - 1)]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + /* dict[key] = value */ + // Do not DECREF INPUTS because the function steals the references + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2( + (PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(key), + PyStackRef_AsPyObjectSteal(value) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto pop_2_error; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_CLASS); - _PyStackRef subject; - _PyStackRef type; - _PyStackRef names; - _PyStackRef attrs; - names = stack_pointer[-1]; - type = stack_pointer[-2]; - subject = stack_pointer[-3]; - // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or - // None on failure. - assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attrs_o = _PyEval_MatchClass(tstate, - PyStackRef_AsPyObjectBorrow(subject), - PyStackRef_AsPyObjectBorrow(type), oparg, - PyStackRef_AsPyObjectBorrow(names)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(subject); - PyStackRef_CLOSE(type); - PyStackRef_CLOSE(names); - if (attrs_o) { - assert(PyTuple_CheckExact(attrs_o)); // Success! - attrs = PyStackRef_FromPyObjectSteal(attrs_o); - } - else { - if (_PyErr_Occurred(tstate)) CEVAL_GOTO(pop_3_error); - // Error! - attrs = PyStackRef_None; // Failure! + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_CLASS); + _PyStackRef subject; + _PyStackRef type; + _PyStackRef names; + _PyStackRef attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; + // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or + // None on failure. + assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attrs_o = _PyEval_MatchClass(tstate, + PyStackRef_AsPyObjectBorrow(subject), + PyStackRef_AsPyObjectBorrow(type), oparg, + PyStackRef_AsPyObjectBorrow(names)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(subject); + PyStackRef_CLOSE(type); + PyStackRef_CLOSE(names); + if (attrs_o) { + assert(PyTuple_CheckExact(attrs_o)); // Success! + attrs = PyStackRef_FromPyObjectSteal(attrs_o); + } + else { + if (_PyErr_Occurred(tstate)) goto pop_3_error; + // Error! + attrs = PyStackRef_None; // Failure! + } + stack_pointer[-3] = attrs; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-3] = attrs; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_KEYS); - _PyStackRef subject; - _PyStackRef keys; - _PyStackRef values_or_none; - keys = stack_pointer[-1]; - subject = stack_pointer[-2]; - // On successful match, PUSH(values). Otherwise, PUSH(None). - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, - PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (values_or_none_o == NULL) CEVAL_GOTO(error); - values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); - stack_pointer[0] = values_or_none; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_KEYS); + _PyStackRef subject; + _PyStackRef keys; + _PyStackRef values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; + // On successful match, PUSH(values). Otherwise, PUSH(None). + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, + PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (values_or_none_o == NULL) goto error; + values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); + stack_pointer[0] = values_or_none; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_MAPPING); - _PyStackRef subject; - _PyStackRef res; - subject = stack_pointer[-1]; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_MAPPING); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_SEQUENCE); - _PyStackRef subject; - _PyStackRef res; - subject = stack_pointer[-1]; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_SEQUENCE); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOP); - DISPATCH(); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOP); + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOT_TAKEN); - DISPATCH(); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOT_TAKEN); + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_EXCEPT); - _PyStackRef exc_value; - exc_value = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_XSETREF(exc_info->exc_value, + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_EXCEPT); + _PyStackRef exc_value; + exc_value = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XSETREF(exc_info->exc_value, PyStackRef_IsNone(exc_value) ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_ITER); - _PyStackRef value; - value = stack_pointer[-1]; - PyStackRef_CLOSE(value); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_ITER); + _PyStackRef value; + value = stack_pointer[-1]; + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_FALSE); - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_FALSE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NONE); - _PyStackRef value; - _PyStackRef b; - _PyStackRef cond; - /* Skip 1 cache entry */ - // _IS_NONE { - value = stack_pointer[-1]; - if (PyStackRef_IsNone(value)) { - b = PyStackRef_True; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NONE); + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + { + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; + } + else { + b = PyStackRef_False; + PyStackRef_CLOSE(value); + } } - else { - b = PyStackRef_False; - PyStackRef_CLOSE(value); + // _POP_JUMP_IF_TRUE + { + cond = b; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - // _POP_JUMP_IF_TRUE - { - cond = b; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); - _PyStackRef value; - _PyStackRef b; - _PyStackRef cond; - /* Skip 1 cache entry */ - // _IS_NONE { - value = stack_pointer[-1]; - if (PyStackRef_IsNone(value)) { - b = PyStackRef_True; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + { + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; + } + else { + b = PyStackRef_False; + PyStackRef_CLOSE(value); + } } - else { - b = PyStackRef_False; - PyStackRef_CLOSE(value); + // _POP_JUMP_IF_FALSE + { + cond = b; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - // _POP_JUMP_IF_FALSE + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS){ { - cond = b; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_TRUE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsFalse(cond); + int flag = PyStackRef_IsTrue(cond); RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_TRUE); - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_TOP); - _PyStackRef value; - value = stack_pointer[-1]; - PyStackRef_CLOSE(value); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_TOP); + _PyStackRef value; + value = stack_pointer[-1]; + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_EXC_INFO); - _PyStackRef exc; - _PyStackRef prev_exc; - _PyStackRef new_exc; - exc = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - if (exc_info->exc_value != NULL) { - prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); - } - else { - prev_exc = PyStackRef_None; - } - assert(PyStackRef_ExceptionInstanceCheck(exc)); - exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); - new_exc = exc; - stack_pointer[-1] = prev_exc; - stack_pointer[0] = new_exc; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_EXC_INFO); + _PyStackRef exc; + _PyStackRef prev_exc; + _PyStackRef new_exc; + exc = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); + } + else { + prev_exc = PyStackRef_None; + } + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_NULL); - _PyStackRef res; - res = PyStackRef_NULL; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_NULL); + _PyStackRef res; + res = PyStackRef_NULL; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(RAISE_VARARGS); - _PyStackRef *args; - args = &stack_pointer[-oparg]; - assert(oparg < 3); - PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; - PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = do_raise(tstate, exc, cause); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - assert(oparg == 0); + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(RAISE_VARARGS); + _PyStackRef *args; + args = &stack_pointer[-oparg]; + assert(oparg < 3); + PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; + PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_reraise(tstate, frame, this_instr); + int err = do_raise(tstate, exc, cause); stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(exception_unwind); - } - CEVAL_GOTO(error); + if (err) { + assert(oparg == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto exception_unwind; + } + goto error; + } + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(RERAISE); - _PyStackRef *values; - _PyStackRef exc_st; - exc_st = stack_pointer[-1]; - values = &stack_pointer[-1 - oparg]; - PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); - assert(oparg >= 0 && oparg <= 2); - if (oparg) { - PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]); - if (PyLong_Check(lasti)) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyLong_AsLong(lasti); - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(!_PyErr_Occurred(tstate)); - } - else { - stack_pointer += -1; + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(RERAISE); + _PyStackRef *values; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); + assert(oparg >= 0 && oparg <= 2); + if (oparg) { + PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]); + if (PyLong_Check(lasti)) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyLong_AsLong(lasti); + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!_PyErr_Occurred(tstate)); + } + else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(exc); + goto error; + } + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); - stack_pointer = _PyFrame_GetStackPointer(frame); - Py_DECREF(exc); - CEVAL_GOTO(error); } - stack_pointer += 1; + assert(exc && PyExceptionInstance_Check(exc)); + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - } - assert(exc && PyExceptionInstance_Check(exc)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(exception_unwind); - DISPATCH(); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto exception_unwind; + } + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESERVED); - assert(0 && "Executing RESERVED instruction."); - Py_FatalError("Executing RESERVED instruction."); - DISPATCH(); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESERVED); + assert(0 && "Executing RESERVED instruction."); + Py_FatalError("Executing RESERVED instruction."); + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME); - PREDICTED(RESUME); - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; - // _LOAD_BYTECODE { - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME); + PREDICTED(RESUME); + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) CEVAL_GOTO(error); - _PyFrame_SetStackPointer(frame, stack_pointer); - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; - frame->instr_ptr = bytecode + off; - // Make sure this_instr gets reset correctley for any uops that - // follow - next_instr = frame->instr_ptr; - DISPATCH(); - } - #endif - } - // _MAYBE_INSTRUMENT - { - if (tstate->tracing == 0) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - CEVAL_GOTO(error); - } - next_instr = this_instr; + if (bytecode == NULL) goto error; + _PyFrame_SetStackPointer(frame, stack_pointer); + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + // Make sure this_instr gets reset correctley for any uops that + // follow + next_instr = frame->instr_ptr; DISPATCH(); } + #endif } - } - // _QUICKEN_RESUME - { - #if ENABLE_SPECIALIZATION_FT - if (tstate->tracing == 0 && this_instr->op.code == RESUME) { - FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + // _MAYBE_INSTRUMENT + { + if (tstate->tracing == 0) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } + next_instr = this_instr; + DISPATCH(); + } + } } - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _CHECK_PERIODIC_IF_NOT_YIELD_FROM - { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) CEVAL_GOTO(error); + // _QUICKEN_RESUME + { + #if ENABLE_SPECIALIZATION_FT + if (tstate->tracing == 0 && this_instr->op.code == RESUME) { + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + } + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + } } } } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME_CHECK); - static_assert(0 == 0, "incorrect cache size"); - #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; - #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); - #ifdef Py_GIL_DISABLED - DEOPT_IF(frame->tlbc_index != + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME_CHECK); + static_assert(0 == 0, "incorrect cache size"); + #if defined(__EMSCRIPTEN__) + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + DEOPT_IF(eval_breaker != version, RESUME); + #ifdef Py_GIL_DISABLED + DEOPT_IF(frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME); - #endif + #endif + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_GENERATOR); - _PyStackRef res; - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (gen == NULL) CEVAL_GOTO(error); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_GENERATOR); + _PyStackRef res; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (gen == NULL) goto error; + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectSteal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_VALUE); - _PyStackRef retval; - _PyStackRef res; - retval = stack_pointer[-1]; - #if TIER_ONE - assert(!frame->is_entry_frame); - #endif - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(EMPTY()); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(frame->return_offset); - res = temp; - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_VALUE); + _PyStackRef retval; + _PyStackRef res; + retval = stack_pointer[-1]; + #if TIER_ONE + assert(!frame->is_entry_frame); + #endif + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(frame->return_offset); + res = temp; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND); - PREDICTED(SEND); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef receiver; - _PyStackRef v; - _PyStackRef retval; - // _SPECIALIZE_SEND { - receiver = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Send(receiver, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND); + PREDICTED(SEND); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef receiver; + _PyStackRef v; + _PyStackRef retval; + // _SPECIALIZE_SEND + { + receiver = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Send(receiver, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(SEND); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } - OPCODE_DEFERRED_INC(SEND); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _SEND - { - v = stack_pointer[-1]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - PyObject *retval_o; - assert(!frame->is_entry_frame); - if ((tstate->interp->eval_frame == NULL) && + // _SEND + { + v = stack_pointer[-1]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + PyObject *retval_o; + assert(!frame->is_entry_frame); + if ((tstate->interp->eval_frame == NULL) && (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver_o; + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2 + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2 + oparg); + assert(gen_frame->previous == NULL); + gen_frame->previous = frame; + DISPATCH_INLINED(gen_frame); + } + if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + retval_o = PyObject_CallMethodOneArg(receiver_o, + &_Py_ID(send), + PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + if (retval_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyGen_FetchStopIterationValue(&retval_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err == 0) { + assert(retval_o != NULL); + JUMPBY(oparg); + } + else { + PyStackRef_CLOSE(v); + goto pop_1_error; + } + } + PyStackRef_CLOSE(v); + retval = PyStackRef_FromPyObjectSteal(retval_o); + } + stack_pointer[-1] = retval; + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS){ + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_GEN); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef receiver; + _PyStackRef v; + _PyInterpreterFrame *gen_frame; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, SEND); + } + // _SEND_GEN_FRAME { - PyGenObject *gen = (PyGenObject *)receiver_o; - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - STACK_SHRINK(1); + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + STAT_INC(SEND, hit); + gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, v); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; assert( 2 + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)( 2 + oparg); - assert(gen_frame->previous == NULL); gen_frame->previous = frame; - DISPATCH_INLINED(gen_frame); } - if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { + // _PUSH_FRAME + { + new_frame = gen_frame; + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS){ + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SETUP_ANNOTATIONS); + PyObject *ann_dict; + if (LOCALS() == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = PyObject_CallMethodOneArg(receiver_o, - &_Py_ID(send), - PyStackRef_AsPyObjectBorrow(v)); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when setting up annotations"); stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; } - if (retval_o == NULL) { + /* check if __annotations__ in locals()... */ + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) goto error; + if (ann_dict == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + ann_dict = PyDict_New(); stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - } + if (ann_dict == NULL) goto error; _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyGen_FetchStopIterationValue(&retval_o); + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err == 0) { - assert(retval_o != NULL); - JUMPBY(oparg); - } - else { - PyStackRef_CLOSE(v); - CEVAL_GOTO(pop_1_error); - } + Py_DECREF(ann_dict); + if (err) goto error; + } + else { + Py_DECREF(ann_dict); } - PyStackRef_CLOSE(v); - retval = PyStackRef_FromPyObjectSteal(retval_o); } - stack_pointer[-1] = retval; DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND_GEN); - static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); - _PyStackRef receiver; - _PyStackRef v; - _PyInterpreterFrame *gen_frame; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, SEND); - } - // _SEND_GEN_FRAME +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS){ { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_ADD); + _PyStackRef set; + _PyStackRef v; v = stack_pointer[-1]; - receiver = stack_pointer[-2]; - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); - STAT_INC(SEND, hit); - gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert( 2 + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2 + oparg); - gen_frame->previous = frame; - } - // _PUSH_FRAME - { - new_frame = gen_frame; - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - DISPATCH(); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SETUP_ANNOTATIONS); - PyObject *ann_dict; - if (LOCALS() == NULL) { + set = stack_pointer[-2 - (oparg-1)]; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when setting up annotations"); - stack_pointer = _PyFrame_GetStackPointer(frame); - CEVAL_GOTO(error); - } - /* check if __annotations__ in locals()... */ - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) CEVAL_GOTO(error); - if (ann_dict == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - ann_dict = PyDict_New(); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (ann_dict == NULL) CEVAL_GOTO(error); - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); + int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); - Py_DECREF(ann_dict); - if (err) CEVAL_GOTO(error); - } - else { - Py_DECREF(ann_dict); + PyStackRef_CLOSE(v); + if (err) goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_ADD); - _PyStackRef set; - _PyStackRef v; - v = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), - PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - if (err) CEVAL_GOTO(pop_1_error); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); - _PyStackRef attr_st; - _PyStackRef func_in; - _PyStackRef func_out; - func_in = stack_pointer[-1]; - attr_st = stack_pointer[-2]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); - PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); - func_out = func_in; - assert(PyFunction_Check(func)); - size_t offset = _Py_FunctionAttributeOffsets[oparg]; - assert(offset != 0); - PyObject **ptr = (PyObject **)(((char *)func) + offset); - assert(*ptr == NULL); - *ptr = attr; - stack_pointer[-2] = func_out; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); + _PyStackRef attr_st; + _PyStackRef func_in; + _PyStackRef func_out; + func_in = stack_pointer[-1]; + attr_st = stack_pointer[-2]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); + PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); + func_out = func_in; + assert(PyFunction_Check(func)); + size_t offset = _Py_FunctionAttributeOffsets[oparg]; + assert(offset != 0); + PyObject **ptr = (PyObject **)(((char *)func) + offset); + assert(*ptr == NULL); + *ptr = attr; + stack_pointer[-2] = func_out; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_UPDATE); - _PyStackRef set; - _PyStackRef iterable; - iterable = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_UPDATE); + _PyStackRef set; + _PyStackRef iterable; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(iterable); - if (err < 0) CEVAL_GOTO(pop_1_error); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(iterable); + if (err < 0) goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR); - PREDICTED(STORE_ATTR); - _Py_CODEUNIT* const this_instr = next_instr - 5; - (void)this_instr; - _PyStackRef owner; - _PyStackRef v; - // _SPECIALIZE_STORE_ATTR { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR); + PREDICTED(STORE_ATTR); + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + _PyStackRef owner; + _PyStackRef v; + // _SPECIALIZE_STORE_ATTR + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_StoreAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(STORE_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 3 cache entries */ + // _STORE_ATTR + { + v = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_StoreAttr(owner, next_instr, name); + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(owner); + if (err) goto pop_2_error; } - OPCODE_DEFERRED_INC(STORE_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 3 cache entries */ - // _STORE_ATTR - { - v = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), - name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(owner); - if (err) CEVAL_GOTO(pop_2_error); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION_AND_LOCK { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(type_version != 0); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); - PyTypeObject *tp = Py_TYPE(owner_o); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION_AND_LOCK + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + DEOPT_IF(true, STORE_ATTR); + } } - } - // _GUARD_DORV_NO_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (_PyObject_GetManagedDict(owner_o) || + // _GUARD_DORV_NO_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + DEOPT_IF(true, STORE_ATTR); + } + } + // _STORE_ATTR_INSTANCE_VALUE + { + value = stack_pointer[-2]; + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + STAT_INC(STORE_ATTR, hit); + assert(_PyObject_GetManagedDict(owner_o) == NULL); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *old_value = *value_ptr; + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); + if (old_value == NULL) { + PyDictValues *values = _PyObject_InlineValues(owner_o); + Py_ssize_t index = value_ptr - values->values; + _PyDictValues_AddToInsertionOrder(values, index); + } UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR); + Py_XDECREF(old_value); + PyStackRef_CLOSE(owner); } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } - // _STORE_ATTR_INSTANCE_VALUE - { - value = stack_pointer[-2]; - uint16_t offset = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - STAT_INC(STORE_ATTR, hit); - assert(_PyObject_GetManagedDict(owner_o) == NULL); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *old_value = *value_ptr; - FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); - if (old_value == NULL) { - PyDictValues *values = _PyObject_InlineValues(owner_o); - Py_ssize_t index = value_ptr - values->values; - _PyDictValues_AddToInsertionOrder(values, index); - } - UNLOCK_OBJECT(owner_o); - Py_XDECREF(old_value); - PyStackRef_CLOSE(owner); - } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_SLOT); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); - } - // _STORE_ATTR_SLOT { - value = stack_pointer[-2]; - uint16_t index = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); - char *addr = (char *)owner_o + index; - STAT_INC(STORE_ATTR, hit); - PyObject *old_value = *(PyObject **)addr; - FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); - UNLOCK_OBJECT(owner_o); - Py_XDECREF(old_value); - PyStackRef_CLOSE(owner); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + } + // _STORE_ATTR_SLOT + { + value = stack_pointer[-2]; + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + char *addr = (char *)owner_o + index; + STAT_INC(STORE_ATTR, hit); + PyObject *old_value = *(PyObject **)addr; + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); + Py_XDECREF(old_value); + PyStackRef_CLOSE(owner); + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); - } - // _STORE_ATTR_WITH_HINT { - value = stack_pointer[-2]; - uint16_t hint = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, STORE_ATTR); - DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR); - #ifdef Py_GIL_DISABLED - if (dict != _PyObject_GetManagedDict(owner_o)) { - UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); } - #endif - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (hint >= (size_t)dict->ma_keys->dk_nentries || + // _STORE_ATTR_WITH_HINT + { + value = stack_pointer[-2]; + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + DEOPT_IF(dict == NULL, STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR); + #ifdef Py_GIL_DISABLED + if (dict != _PyObject_GetManagedDict(owner_o)) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } + #endif + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (hint >= (size_t)dict->ma_keys->dk_nentries || !DK_IS_UNICODE(dict->ma_keys)) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } + PyObject *old_value = ep->me_value; + if (old_value == NULL) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); - } - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) { - UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); - } - PyObject *old_value = ep->me_value; - if (old_value == NULL) { - UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, + // when dict only holds the strong reference to value in ep->me_value. + Py_XDECREF(old_value); + STAT_INC(STORE_ATTR, hit); + PyStackRef_CLOSE(owner); } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); - UNLOCK_OBJECT(dict); - // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, - // when dict only holds the strong reference to value in ep->me_value. - Py_XDECREF(old_value); - STAT_INC(STORE_ATTR, hit); - PyStackRef_CLOSE(owner); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_DEREF); - _PyStackRef v; - v = stack_pointer[-1]; - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_DEREF); + _PyStackRef v; + v = stack_pointer[-1]; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST); - _PyStackRef value; - value = stack_pointer[-1]; - SETLOCAL(oparg, value); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST); + _PyStackRef value; + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); - _PyStackRef value1; - _PyStackRef value2; - value1 = stack_pointer[-1]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - SETLOCAL(oparg1, value1); - value2 = PyStackRef_DUP(GETLOCAL(oparg2)); - stack_pointer[-1] = value2; - DISPATCH(); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); + _PyStackRef value1; + _PyStackRef value2; + value1 = stack_pointer[-1]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[-1] = value2; + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_STORE_FAST); - _PyStackRef value2; - _PyStackRef value1; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - SETLOCAL(oparg1, value1); - SETLOCAL(oparg2, value2); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_STORE_FAST); + _PyStackRef value2; + _PyStackRef value1; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + SETLOCAL(oparg2, value2); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_GLOBAL); - _PyStackRef v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - if (err) CEVAL_GOTO(pop_1_error); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_NAME); - _PyStackRef v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when storing %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - CEVAL_GOTO(pop_1_error); - } - if (PyDict_CheckExact(ns)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_GLOBAL); + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + if (err) goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(v); - if (err) CEVAL_GOTO(pop_1_error); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_SLICE); - _PyStackRef v; - _PyStackRef container; - _PyStackRef start; - _PyStackRef stop; - // _SPECIALIZE_STORE_SLICE - { - // Placeholder until we implement STORE_SLICE specialization - #if ENABLE_SPECIALIZATION - OPCODE_DEFERRED_INC(STORE_SLICE); - #endif /* ENABLE_SPECIALIZATION */ - } - // _STORE_SLICE +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS){ { - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - v = stack_pointer[-4]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); - stack_pointer = _PyFrame_GetStackPointer(frame); + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_NAME); + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); int err; - if (slice == NULL) { - err = 1; + if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when storing %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + goto pop_1_error; + } + if (PyDict_CheckExact(ns)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); } else { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); - Py_DECREF(slice); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); } PyStackRef_CLOSE(v); - PyStackRef_CLOSE(container); - if (err) CEVAL_GOTO(pop_4_error); + if (err) goto pop_1_error; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR); - PREDICTED(STORE_SUBSCR); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef container; - _PyStackRef sub; - _PyStackRef v; - // _SPECIALIZE_STORE_SUBSCR +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS){ { - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_SLICE); + _PyStackRef v; + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + // _SPECIALIZE_STORE_SLICE + { + // Placeholder until we implement STORE_SLICE specialization + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(STORE_SLICE); + #endif /* ENABLE_SPECIALIZATION */ + } + // _STORE_SLICE + { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_StoreSubscr(container, sub, next_instr); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + int err; + if (slice == NULL) { + err = 1; + } + else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + Py_DECREF(slice); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); + if (err) goto pop_4_error; } - OPCODE_DEFERRED_INC(STORE_SUBSCR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ + stack_pointer += -4; + assert(WITHIN_STACK_BOUNDS()); } - // _STORE_SUBSCR + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS){ { - v = stack_pointer[-3]; - /* container[sub] = v */ - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (err) CEVAL_GOTO(pop_3_error); + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR); + PREDICTED(STORE_SUBSCR); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef container; + _PyStackRef sub; + _PyStackRef v; + // _SPECIALIZE_STORE_SUBSCR + { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_StoreSubscr(container, sub, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(STORE_SUBSCR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _STORE_SUBSCR + { + v = stack_pointer[-3]; + /* container[sub] = v */ + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (err) goto pop_3_error; + } + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_DICT); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef dict_st; - _PyStackRef sub; - /* Skip 1 cache entry */ - sub = stack_pointer[-1]; - dict_st = stack_pointer[-2]; - value = stack_pointer[-3]; - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); - STAT_INC(STORE_SUBSCR, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(sub), - PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(dict_st); - if (err) CEVAL_GOTO(pop_3_error); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef dict_st; + _PyStackRef sub; + /* Skip 1 cache entry */ + sub = stack_pointer[-1]; + dict_st = stack_pointer[-2]; + value = stack_pointer[-3]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + STAT_INC(STORE_SUBSCR, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(sub), + PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(dict_st); + if (err) goto pop_3_error; + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef list_st; - _PyStackRef sub_st; - /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - value = stack_pointer[-3]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); - // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR); - // Ensure index < len(list) - if (index >= PyList_GET_SIZE(list)) { - UNLOCK_OBJECT(list); - DEOPT_IF(true, STORE_SUBSCR); - } - STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); - PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); - assert(old_value != NULL); - UNLOCK_OBJECT(list); // unlock before decrefs! - Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - PyStackRef_CLOSE(list_st); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef list_st; + _PyStackRef sub_st; + /* Skip 1 cache entry */ + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + value = stack_pointer[-3]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + // Ensure nonnegative, zero-or-one-digit ints. + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR); + // Ensure index < len(list) + if (index >= PyList_GET_SIZE(list)) { + UNLOCK_OBJECT(list); + DEOPT_IF(true, STORE_SUBSCR); + } + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); + assert(old_value != NULL); + UNLOCK_OBJECT(list); // unlock before decrefs! + Py_DECREF(old_value); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(list_st); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SWAP); - _PyStackRef bottom_in; - _PyStackRef top_in; - _PyStackRef top_out; - _PyStackRef bottom_out; - top_in = stack_pointer[-1]; - bottom_in = stack_pointer[-2 - (oparg-2)]; - bottom_out = bottom_in; - top_out = top_in; - assert(oparg >= 2); - stack_pointer[-2 - (oparg-2)] = top_out; - stack_pointer[-1] = bottom_out; - DISPATCH(); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SWAP); + _PyStackRef bottom_in; + _PyStackRef top_in; + _PyStackRef top_out; + _PyStackRef bottom_out; + top_in = stack_pointer[-1]; + bottom_in = stack_pointer[-2 - (oparg-2)]; + bottom_out = bottom_in; + top_out = top_in; + assert(oparg >= 2); + stack_pointer[-2 - (oparg-2)] = top_out; + stack_pointer[-1] = bottom_out; + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL); - PREDICTED(TO_BOOL); - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - _PyStackRef value; - _PyStackRef res; - // _SPECIALIZE_TO_BOOL { - value = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL); + PREDICTED(TO_BOOL); + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + _PyStackRef value; + _PyStackRef res; + // _SPECIALIZE_TO_BOOL + { + value = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ToBool(value, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(TO_BOOL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _TO_BOOL + { _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ToBool(value, next_instr); + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(TO_BOOL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _TO_BOOL - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (err < 0) CEVAL_GOTO(pop_1_error); - res = err ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[-1] = res; - DISPATCH(); + PyStackRef_CLOSE(value); + if (err < 0) goto pop_1_error; + res = err ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-1] = res; + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS){ - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL); - } - // _REPLACE_WITH_TRUE { - value = owner; - PyStackRef_CLOSE(value); - res = PyStackRef_True; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL); + } + // _REPLACE_WITH_TRUE + { + value = owner; + PyStackRef_CLOSE(value); + res = PyStackRef_True; + } + stack_pointer[-1] = res; } - stack_pointer[-1] = res; DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_BOOL); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - DISPATCH(); + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_BOOL); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_INT); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL); - STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value_o)) { - assert(_Py_IsImmortal(value_o)); - res = PyStackRef_False; - } - else { - PyStackRef_CLOSE(value); - res = PyStackRef_True; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_INT); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL); + STAT_INC(TO_BOOL, hit); + if (_PyLong_IsZero((PyLongObject *)value_o)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; + } + else { + PyStackRef_CLOSE(value); + res = PyStackRef_True; + } + stack_pointer[-1] = res; } - stack_pointer[-1] = res; DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_LIST); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL); - STAT_INC(TO_BOOL, hit); - res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; - PyStackRef_CLOSE(value); - stack_pointer[-1] = res; - DISPATCH(); + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_LIST); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL); + STAT_INC(TO_BOOL, hit); + res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; + PyStackRef_CLOSE(value); + stack_pointer[-1] = res; + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_NONE); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - res = PyStackRef_False; - stack_pointer[-1] = res; - DISPATCH(); + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_NONE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + // This one is a bit weird, because we expect *some* failures: + DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + res = PyStackRef_False; + stack_pointer[-1] = res; + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_STR); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL); - STAT_INC(TO_BOOL, hit); - if (value_o == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value_o)); - res = PyStackRef_False; - } - else { - assert(Py_SIZE(value_o)); - PyStackRef_CLOSE(value); - res = PyStackRef_True; + { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_STR); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL); + STAT_INC(TO_BOOL, hit); + if (value_o == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; + } + else { + assert(Py_SIZE(value_o)); + PyStackRef_CLOSE(value); + res = PyStackRef_True; + } + stack_pointer[-1] = res; } - stack_pointer[-1] = res; DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_INVERT); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (res_o == NULL) CEVAL_GOTO(pop_1_error); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; - DISPATCH(); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_INVERT); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) goto pop_1_error; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-1] = res; + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NEGATIVE); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (res_o == NULL) CEVAL_GOTO(pop_1_error); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; - DISPATCH(); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NEGATIVE); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) goto pop_1_error; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-1] = res; + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NOT); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(value)); - res = PyStackRef_IsFalse(value) - ? PyStackRef_True : PyStackRef_False; - stack_pointer[-1] = res; - DISPATCH(); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NOT); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + stack_pointer[-1] = res; + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNPACK_EX); - _PyStackRef seq; - _PyStackRef *right; - seq = stack_pointer[-1]; - right = &stack_pointer[(oparg & 0xFF)]; - _PyStackRef *top = right + (oparg >> 8); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(seq); - if (res == 0) CEVAL_GOTO(pop_1_error); - stack_pointer += (oparg & 0xFF) + (oparg >> 8); - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNPACK_EX); + _PyStackRef seq; + _PyStackRef *right; + seq = stack_pointer[-1]; + right = &stack_pointer[(oparg & 0xFF)]; + _PyStackRef *top = right + (oparg >> 8); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(seq); + if (res == 0) goto pop_1_error; + stack_pointer += (oparg & 0xFF) + (oparg >> 8); + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE); - PREDICTED(UNPACK_SEQUENCE); - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef seq; - _PyStackRef *output; - // _SPECIALIZE_UNPACK_SEQUENCE { - seq = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE); + PREDICTED(UNPACK_SEQUENCE); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef seq; + _PyStackRef *output; + // _SPECIALIZE_UNPACK_SEQUENCE + { + seq = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + (void)seq; + (void)counter; + } + // _UNPACK_SEQUENCE + { + output = &stack_pointer[-1]; + _PyStackRef *top = output + oparg; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + PyStackRef_CLOSE(seq); + if (res == 0) goto pop_1_error; } - OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - (void)seq; - (void)counter; - } - // _UNPACK_SEQUENCE - { - output = &stack_pointer[-1]; - _PyStackRef *top = output + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(seq); - if (res == 0) CEVAL_GOTO(pop_1_error); + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef seq; - _PyStackRef *values; - /* Skip 1 cache entry */ - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE); - if (PyList_GET_SIZE(seq_o) != oparg) { + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef seq; + _PyStackRef *values; + /* Skip 1 cache entry */ + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE); + if (PyList_GET_SIZE(seq_o) != oparg) { + UNLOCK_OBJECT(seq_o); + DEOPT_IF(true, UNPACK_SEQUENCE); + } + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyList_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } UNLOCK_OBJECT(seq_o); - DEOPT_IF(true, UNPACK_SEQUENCE); - } - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectNew(items[i]); + PyStackRef_CLOSE(seq); + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } - UNLOCK_OBJECT(seq_o); - PyStackRef_CLOSE(seq); - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef seq; - _PyStackRef *values; - /* Skip 1 cache entry */ - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE); - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectNew(items[i]); - } - PyStackRef_CLOSE(seq); - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef seq; + _PyStackRef *values; + /* Skip 1 cache entry */ + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + PyStackRef_CLOSE(seq); + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef seq; - _PyStackRef val1; - _PyStackRef val0; - /* Skip 1 cache entry */ - seq = stack_pointer[-1]; - assert(oparg == 2); - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); - STAT_INC(UNPACK_SEQUENCE, hit); - val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); - val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - PyStackRef_CLOSE(seq); - stack_pointer[-1] = val1; - stack_pointer[0] = val0; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + /* Skip 1 cache entry */ + seq = stack_pointer[-1]; + assert(oparg == 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + PyStackRef_CLOSE(seq); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(WITH_EXCEPT_START); - _PyStackRef exit_func; - _PyStackRef exit_self; - _PyStackRef lasti; - _PyStackRef val; - _PyStackRef res; - val = stack_pointer[-1]; - lasti = stack_pointer[-3]; - exit_self = stack_pointer[-4]; - exit_func = stack_pointer[-5]; - /* At the top of the stack are 4 values: - - val: TOP = exc_info() - - unused: SECOND = previous exception - - lasti: THIRD = lasti of exception in exc_info() - - exit_self: FOURTH = the context or NULL - - exit_func: FIFTH = the context.__exit__ function or context.__exit__ bound method - We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). - Then we push the __exit__ return value. - */ - PyObject *exc, *tb; - PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); - PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); - assert(val_o && PyExceptionInstance_Check(val_o)); - exc = PyExceptionInstance_Class(val_o); - tb = PyException_GetTraceback(val_o); - if (tb == NULL) { - tb = Py_None; - } - else { - Py_DECREF(tb); - } - assert(PyStackRef_LongCheck(lasti)); - (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; - int has_self = !PyStackRef_IsNull(exit_self); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, - (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) CEVAL_GOTO(error); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(WITH_EXCEPT_START); + _PyStackRef exit_func; + _PyStackRef exit_self; + _PyStackRef lasti; + _PyStackRef val; + _PyStackRef res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_self = stack_pointer[-4]; + exit_func = stack_pointer[-5]; + /* At the top of the stack are 4 values: + - val: TOP = exc_info() + - unused: SECOND = previous exception + - lasti: THIRD = lasti of exception in exc_info() + - exit_self: FOURTH = the context or NULL + - exit_func: FIFTH = the context.__exit__ function or context.__exit__ bound method + We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). + Then we push the __exit__ return value. + */ + PyObject *exc, *tb; + PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); + PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); + assert(val_o && PyExceptionInstance_Check(val_o)); + exc = PyExceptionInstance_Class(val_o); + tb = PyException_GetTraceback(val_o); + if (tb == NULL) { + tb = Py_None; + } + else { + Py_DECREF(tb); + } + assert(PyStackRef_LongCheck(lasti)); + (void)lasti; // Shut up compiler warning if asserts are off + PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + int has_self = !PyStackRef_IsNull(exit_self); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) goto error; + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS){ - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(YIELD_VALUE); - _PyStackRef retval; - _PyStackRef value; - retval = stack_pointer[-1]; - // NOTE: It's important that YIELD_VALUE never raises an exception! - // The compiler treats any exception raised here as a failed close() - // or throw() call. - #if TIER_ONE - assert(!frame->is_entry_frame); - #endif - frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(YIELD_VALUE); + _PyStackRef retval; + _PyStackRef value; + retval = stack_pointer[-1]; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + #if TIER_ONE + assert(!frame->is_entry_frame); + #endif + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - value = temp; - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + value = temp; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS){ + { _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", _PyFrame_GetCode(frame)->co_filename, PyUnstable_InterpreterFrame_GetLine(frame), opcode); - CEVAL_GOTO(error);} + } + TAIL_CALL(error); +} static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [BINARY_OP] = _TAIL_CALL_BINARY_OP, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index e9286d180a0b05..0035deb02ffdf0 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -197,21 +197,18 @@ def error_if( except ValueError: offset = -1 if offset > 0: - self.out.emit(f"CEVAL_GOTO(pop_{offset}_") + self.out.emit(f"goto pop_{offset}_") self.out.emit(label) - self.out.emit(")") self.out.emit(";\n") elif offset == 0: - self.out.emit("CEVAL_GOTO(") + self.out.emit("goto ") self.out.emit(label) - self.out.emit(")") self.out.emit(";\n") else: self.out.emit("{\n") storage.copy().flush(self.out) - self.out.emit("CEVAL_GOTO(") + self.out.emit("goto ") self.out.emit(label) - self.out.emit(")") self.out.emit(";\n") self.out.emit("}\n") return not unconditional @@ -227,7 +224,7 @@ def error_no_pop( next(tkn_iter) # LPAREN next(tkn_iter) # RPAREN next(tkn_iter) # Semi colon - self.out.emit_at("CEVAL_GOTO(error);", tkn) + self.out.emit_at("goto error;", tkn) return False def decref_inputs( diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index dca3e53f9dce04..2f0da243f97da9 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -33,7 +33,7 @@ TARGET_LABEL = "TAIL_CALL_TARGET" -def generate_label_handlers(infile: TextIO, outfile: TextIO) -> None: +def generate_label_handlers(infile: TextIO, outfile: TextIO) -> list[str]: out = CWriter(outfile, 0, False) str_in = infile.read() # https://stackoverflow.com/questions/8303488/regex-to-match-any-character-including-new-lines @@ -54,14 +54,15 @@ def generate_label_handlers(infile: TextIO, outfile: TextIO) -> None: if TARGET_LABEL in line: break if label := re.findall(r"goto (\w+);", line): - out.emit(f"CEVAL_GOTO({label[0]});\n") + out.emit(f"TAIL_CALL({label[0]});\n") else: out.emit_text(line) out.emit("\n") if fallthrough_proto: - out.emit(f"CEVAL_GOTO({fallthrough_proto});\n") + out.emit(f"TAIL_CALL({fallthrough_proto});\n") out.emit("}\n") out.emit("\n") + return function_protos # For unit testing. def generate_label_handlers_from_files( @@ -93,7 +94,7 @@ def generate_tier1( out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256];\n"); with open(DEFAULT_CEVAL_INPUT, "r") as infile: - generate_label_handlers(infile, outfile) + err_labels = generate_label_handlers(infile, outfile) emitter = Emitter(out) out.emit("\n") @@ -101,9 +102,23 @@ def generate_tier1( out.emit("\n") out.emit(function_proto(name)) out.emit("{\n") + # We wrap this with a block to signal to GCC that the local variables + # are dead at the tail call site. + # Otherwise, GCC 15's escape analysis may think there are + # escaping locals. + # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=118430#c1 + out.emit("{\n") write_single_inst(out, emitter, name, inst) + out.emit("}\n") if not inst.parts[-1].properties.always_exits: out.emit("DISPATCH();\n") + # Note: this produces 2 jumps, but a tail call directly + # at the branch also produces the same. + # Furthermore, this is required to make GCC 15's escape analysis happy + # as written above. + for err_label in err_labels: + out.emit(f"{err_label}:\n") + out.emit(f"TAIL_CALL({err_label});\n") out.start_line() out.emit("}\n") @@ -112,6 +127,7 @@ def generate_tier1( # Emit unknown opcode handler. out.emit(function_proto("UNKNOWN_OPCODE")) out.emit("{\n") + out.emit("{\n") out.emit(""" _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", @@ -119,7 +135,8 @@ def generate_tier1( PyUnstable_InterpreterFrame_GetLine(frame), opcode); """) - out.emit("CEVAL_GOTO(error);") + out.emit("}\n") + out.emit("TAIL_CALL(error);\n") out.emit("}\n") out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256] = {\n") From e1988942ca26440a0df6f3949e93ddc0dbd1e57e Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 14 Jan 2025 07:19:52 +0800 Subject: [PATCH 056/110] Fix test --- Lib/test/test_generated_cases.py | 24 ++++++++++++------------ Python/ceval.c | 3 ++- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 143bcc7c5586e1..72b7f28f61834e 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -540,7 +540,7 @@ def test_error_if_plain(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - if (cond) CEVAL_GOTO(label); + if (cond) goto label; DISPATCH(); } """ @@ -557,7 +557,7 @@ def test_error_if_plain_with_comment(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - if (cond) CEVAL_GOTO(label); + if (cond) goto label; // Comment is ok DISPATCH(); } @@ -584,7 +584,7 @@ def test_error_if_pop(self): right = stack_pointer[-1]; left = stack_pointer[-2]; SPAM(left, right); - if (cond) CEVAL_GOTO(pop_2_label); + if (cond) goto pop_2_label; res = 0; stack_pointer[-2] = res; stack_pointer += -1; @@ -613,7 +613,7 @@ def test_error_if_pop_with_result(self): right = stack_pointer[-1]; left = stack_pointer[-2]; res = SPAM(left, right); - if (cond) CEVAL_GOTO(pop_2_label); + if (cond) goto pop_2_label; stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -931,7 +931,7 @@ def test_array_error_if(self): if (oparg == 0) { stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(somewhere); + goto somewhere; } stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1394,7 +1394,7 @@ def test_pop_on_error_peeks(self): // THIRD { // Mark j and k as used - if (cond) CEVAL_GOTO(pop_2_error); + if (cond) goto pop_2_error; } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -1437,7 +1437,7 @@ def test_push_then_error(self): stack_pointer[1] = b; stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); - CEVAL_GOTO(error); + goto error; } } stack_pointer[0] = a; @@ -1464,14 +1464,14 @@ def test_error_if_true(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP1); - CEVAL_GOTO(here); + goto here; } TARGET(OP2) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP2); - CEVAL_GOTO(there); + goto there; } """ self.run_cases_test(input, output) @@ -1854,7 +1854,7 @@ def test_fallthrough_label(self): Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS) { DO_THING(); - CEVAL_GOTO(fallthrough); + TAIL_CALL(fallthrough); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_fallthrough(TAIL_CALL_PARAMS) @@ -1888,10 +1888,10 @@ def test_transform_gotos(self): Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS) { if (thing) { - CEVAL_GOTO(fallthrough); + TAIL_CALL(fallthrough); } DO_THING(); - CEVAL_GOTO(fallthrough); + TAIL_CALL(fallthrough); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_fallthrough(TAIL_CALL_PARAMS) diff --git a/Python/ceval.c b/Python/ceval.c index 7ed9ef14c3eec8..e1eadb8b5f7173 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -785,7 +785,8 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) #ifdef Py_TAIL_CALL_INTERP #include "generated_tail_call_handlers.c.h" -static inline PyObject *_TAIL_CALL_shim(TAIL_CALL_PARAMS) +static inline PyObject * +_TAIL_CALL_shim(TAIL_CALL_PARAMS) { opcode = next_instr->op.code; oparg = next_instr->op.arg; From 29b6237c334717b02e86306d5d0ba77822deacbf Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 14 Jan 2025 07:47:21 +0800 Subject: [PATCH 057/110] Fix JIT builds --- Python/bytecodes.c | 2 +- Python/executor_cases.c.h | 2 +- Python/generated_cases.c.h | 2 +- Python/generated_tail_call_handlers.c.h | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index abb99e4fc5447f..987c870db5fa56 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4046,7 +4046,7 @@ dummy_func( PyObject *res_o = PyLong_FromSsize_t(len_i); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { - goto error; + GOTO_ERROR(error); } PyStackRef_CLOSE(arg_stackref); DEAD(args); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 67ba03bdeb1baa..1d965a2afd1564 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4941,7 +4941,7 @@ PyObject *res_o = PyLong_FromSsize_t(len_i); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { - goto error; + GOTO_ERROR(error); } PyStackRef_CLOSE(arg_stackref); PyStackRef_CLOSE(callable[0]); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 01d0962908b175..c10b86ba655593 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2379,7 +2379,7 @@ PyObject *res_o = PyLong_FromSsize_t(len_i); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { - goto error; + GOTO_ERROR(error); } PyStackRef_CLOSE(arg_stackref); PyStackRef_CLOSE(callable[0]); diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 57eb2e7cda4fc7..be358b4bda7cc2 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -3242,7 +3242,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ PyObject *res_o = PyLong_FromSsize_t(len_i); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { - goto error; + GOTO_ERROR(error); } PyStackRef_CLOSE(arg_stackref); PyStackRef_CLOSE(callable[0]); From 92411f0000ab6cf9968f30b5042c36e34c71ea8e Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 14 Jan 2025 15:31:07 +0800 Subject: [PATCH 058/110] fix typo --- Python/ceval_macros.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index bfea96a8ccdccc..efadf30157d3b2 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -47,7 +47,7 @@ #define OR_DTRACE_LINE #endif -#ifdef HAVE_COMPUTED_GOTO +#ifdef HAVE_COMPUTED_GOTOS #ifndef USE_COMPUTED_GOTOS #define USE_COMPUTED_GOTOS 1 #endif From 501f64534c0e14a7bd388ac9af81a1090245ca28 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 14 Jan 2025 15:40:56 +0800 Subject: [PATCH 059/110] half the runners --- .github/workflows/tail-call.yml | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index 8bb290a0275ea8..a6afba39da0d5e 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -38,9 +38,6 @@ jobs: - aarch64-apple-darwin/clang - x86_64-unknown-linux-gnu/gcc - aarch64-unknown-linux-gnu/gcc - debug: - - true - - false llvm: - 19 include: @@ -74,25 +71,25 @@ jobs: with: python-version: '3.11' - - name: Native Windows + - name: Native Windows (Debug) if: runner.os == 'Windows' && matrix.architecture != 'ARM64' run: | choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 - ./PCbuild/build.bat --tail-call-interp ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} - ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + ./PCbuild/build.bat --tail-call-interp -d -p ${{ matrix.architecture }} + ./PCbuild/rt.bat -d -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 # No tests (yet): - - name: Emulated Windows + - name: Emulated Windows (Release) if: runner.os == 'Windows' && matrix.architecture == 'ARM64' run: | choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 - ./PCbuild/build.bat --tail-call-interp ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} + ./PCbuild/build.bat --tail-call-interp -p ${{ matrix.architecture }} # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. # This is a bug in the macOS runner image where the pre-installed Python is installed in the same # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes # the symlink to the pre-installed Python so that the Homebrew Python is used instead. - - name: Native macOS + - name: Native macOS (Debug) if: runner.os == 'macOS' run: | brew update @@ -101,17 +98,17 @@ jobs: export SDKROOT="$(xcrun --show-sdk-path)" export PATH="/opt/homebrew/opt/llvm/bin:$PATH" export PATH="/usr/local/opt/llvm/bin:$PATH" - CC=clang-19 ./configure --with-tail-call-interp ${{ matrix.debug && '--with-pydebug' || '' }} + CC=clang-19 ./configure --with-tail-call-interp --with-pydebug make all --jobs 4 ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - name: Native Linux + - name: Native Linux (Release) # Forks don't have access to our paid AArch64 runners. Skip those: if: runner.os == 'Linux' && (matrix.architecture == 'x86_64' || github.repository_owner == 'python') run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - CC=clang-19 ./configure --with-tail-call-interp ${{ matrix.debug && '--with-pydebug' || '' }} + CC=clang-19 ./configure --with-tail-call-interp make all --jobs 4 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 From 7d33768fbfba280b8b949b29fd8af0cebf80bbfd Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 14 Jan 2025 15:43:16 +0800 Subject: [PATCH 060/110] Update tail-call.yml --- .github/workflows/tail-call.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index a6afba39da0d5e..5310e49af7f3cf 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -22,8 +22,8 @@ concurrency: cancel-in-progress: true jobs: - jit: - name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) + tail-call: + name: ${{ matrix.target }} runs-on: ${{ matrix.runner }} timeout-minutes: 90 strategy: From 2a860a2eded4d87684dde3542432377ce2d016f9 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 16 Jan 2025 13:25:38 +0800 Subject: [PATCH 061/110] remove unused macro --- Tools/jit/template.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 6603d7a412980b..95c90bda70f352 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -49,13 +49,6 @@ goto LABEL ## _tier_two; \ } while (0) -#undef CEVAL_GOTO -#define CEVAL_GOTO(LABEL) \ - do { \ - goto LABEL ## _tier_two; \ - } while (0) - - #undef GOTO_TIER_TWO #define GOTO_TIER_TWO(EXECUTOR) \ do { \ From df5d01ca27e4c0f10d3653fb5c663d24df716996 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 16 Jan 2025 20:44:01 +0800 Subject: [PATCH 062/110] catch up with main --- Python/generated_tail_call_handlers.c.h | 307 ++++++++++++++---------- 1 file changed, 186 insertions(+), 121 deletions(-) diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index be358b4bda7cc2..b323517226bf98 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -741,8 +741,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); - stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); } @@ -1250,15 +1250,17 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS){ err = PySet_Add(set_o, PyStackRef_AsPyObjectBorrow(values[i])); stack_pointer = _PyFrame_GetStackPointer(frame); } - PyStackRef_CLOSE(values[i]); + } + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); } if (err != 0) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(set_o); - { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; } set = PyStackRef_FromPyObjectSteal(set_o); stack_pointer[-oparg] = set; @@ -1506,8 +1508,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS){ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } // Check if the call can be inlined or not @@ -1520,7 +1523,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS){ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable[0], locals, - args, total_args, NULL, frame + arguments, total_args, NULL, frame ); stack_pointer = _PyFrame_GetStackPointer(frame); // Manipulate stack directly since we leave using DISPATCH_INLINED(). @@ -1535,12 +1538,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS){ DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1556,7 +1560,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS){ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); if (res_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _Py_call_instrumentation_exc2( @@ -1576,10 +1580,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS){ } } assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2025,11 +2030,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(arguments[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2127,11 +2132,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARA stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(arguments[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2198,20 +2203,21 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS( /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); STAT_INC(CALL, hit); - /* res = func(self, args, nargs, kwnames) */ + /* res = func(self, arguments, nargs, kwnames) */ _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void)) PyCFunction_GET_FUNCTION(callable_o); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); @@ -2229,11 +2235,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS( stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2758,8 +2764,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); // oparg counts all of the args, but *not* self: int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); @@ -2774,7 +2781,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable[0], locals, - args, positional_args, kwnames_o, frame + arguments, positional_args, kwnames_o, frame ); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(kwnames); @@ -2791,7 +2798,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); @@ -2815,7 +2822,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); if (opcode == INSTRUMENTED_CALL_KW) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); if (res_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _Py_call_instrumentation_exc2( @@ -2834,12 +2841,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ } } } - PyStackRef_CLOSE(kwnames); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); if (res_o == NULL) { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2924,8 +2931,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_P PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); @@ -2936,7 +2944,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_P _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable[0], locals, - args, positional_args, kwnames_o, frame + arguments, positional_args, kwnames_o, frame ); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(kwnames); @@ -3023,12 +3031,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) #endif PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); @@ -3053,10 +3062,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) PyStackRef_CLOSE(kwnames); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -3136,8 +3146,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); @@ -3148,7 +3159,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable[0], locals, - args, positional_args, kwnames_o, frame + arguments, positional_args, kwnames_o, frame ); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(kwnames); @@ -3345,8 +3356,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; @@ -3354,11 +3366,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); - PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); @@ -3378,11 +3390,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -3448,8 +3460,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; @@ -3457,11 +3470,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); @@ -3481,11 +3494,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -3743,12 +3756,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PA #endif PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); @@ -3769,10 +3783,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PA stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -4416,8 +4431,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int res_bool = PyObject_IsTrue(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (res_bool < 0) goto error; res = res_bool ? PyStackRef_True : PyStackRef_False; } @@ -4937,7 +4952,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS){ stack_pointer = _PyFrame_GetStackPointer(frame); goto error; } + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(oldobj); + stack_pointer = _PyFrame_GetStackPointer(frame); } DISPATCH(); pop_4_error: @@ -5710,7 +5727,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS){ #ifndef Py_GIL_DISABLED if (seq != NULL) { it->it_seq = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); } #endif /* Jump forward oparg, then skip following END_FOR instruction */ @@ -5836,7 +5855,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS) if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { if (seq != NULL) { it->it_seq = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); } /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); @@ -5915,8 +5936,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS){ "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter_o)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); goto error; } iter = PyStackRef_FromPyObjectSteal(iter_o); @@ -6294,8 +6315,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARA PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } // Check if the call can be inlined or not @@ -6308,7 +6330,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARA _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable[0], locals, - args, total_args, NULL, frame + arguments, total_args, NULL, frame ); stack_pointer = _PyFrame_GetStackPointer(frame); // Manipulate stack directly since we leave using DISPATCH_INLINED(). @@ -6323,12 +6345,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARA DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -6344,7 +6367,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARA STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); if (res_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _Py_call_instrumentation_exc2( @@ -6364,10 +6387,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARA } } assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - for (int i = 0; i < total_args; i++) { - PyStackRef_CLOSE(args[i]); - } PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -7664,7 +7688,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS @@ -7717,7 +7741,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_C PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR); } // _GUARD_TYPE_VERSION { @@ -7776,7 +7800,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDE DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); @@ -7836,19 +7860,24 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { uint16_t offset = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = *value_ptr; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); DEOPT_IF(attr_o == NULL, LOAD_ATTR); + #ifdef Py_GIL_DISABLED + if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { + DEOPT_IF(true, LOAD_ATTR); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectSteal(attr_o); PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ @@ -7898,7 +7927,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_ { uint16_t dictoffset = read_u16(&this_instr[4].cache); char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = *(PyObject **)ptr; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL, LOAD_ATTR); } @@ -8013,14 +8042,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAI { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&this_instr[4].cache); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); } // _LOAD_ATTR_METHOD_WITH_VALUES { @@ -8197,14 +8228,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VAL { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&this_instr[4].cache); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { @@ -8341,12 +8374,17 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) { uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - char *addr = (char *)owner_o + index; - PyObject *attr_o = *(PyObject **)addr; + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); DEOPT_IF(attr_o == NULL, LOAD_ATTR); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + DEOPT_IF(!increfed, LOAD_ATTR); + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectNew(attr_o); PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ @@ -8381,6 +8419,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; + PyDictObject *dict; _PyStackRef attr; _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ @@ -8396,26 +8435,40 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, LOAD_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); + PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); + DEOPT_IF(dict_o == NULL, LOAD_ATTR); + assert(PyDict_CheckExact((PyObject *)dict_o)); + dict = dict_o; } // _LOAD_ATTR_WITH_HINT { uint16_t hint = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject *attr_o; - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + if (!LOCK_OBJECT(dict)) { + DEOPT_IF(true, LOAD_ATTR); + } + if (hint >= (size_t)dict->ma_keys->dk_nentries) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, LOAD_ATTR); + } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), LOAD_ATTR); + if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, LOAD_ATTR); + } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, LOAD_ATTR); + } attr_o = ep->me_value; - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + if (attr_o == NULL) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, LOAD_ATTR); + } STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); - attr = PyStackRef_FromPyObjectSteal(attr_o); + attr = PyStackRef_FromPyObjectNew(attr_o); + UNLOCK_OBJECT(dict); null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } @@ -9389,8 +9442,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *attr_o = PyObject_GetAttr(super, name); - stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(super); + stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) goto error; attr = PyStackRef_FromPyObjectSteal(attr_o); null = PyStackRef_NULL; @@ -10276,8 +10329,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); - stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(exc); + stack_pointer = _PyFrame_GetStackPointer(frame); goto error; } stack_pointer += 1; @@ -10766,12 +10819,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARA _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; } else { + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); } } DISPATCH(); @@ -11021,11 +11076,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_C _PyDictValues_AddToInsertionOrder(values, index); } UNLOCK_OBJECT(owner_o); - Py_XDECREF(old_value); PyStackRef_CLOSE(owner); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); pop_4_error: @@ -11074,11 +11131,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS PyObject *old_value = *(PyObject **)addr; FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(owner_o); - Py_XDECREF(old_value); PyStackRef_CLOSE(owner); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); pop_4_error: @@ -11155,12 +11214,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P UNLOCK_OBJECT(dict); // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. - Py_XDECREF(old_value); STAT_INC(STORE_ATTR, hit); PyStackRef_CLOSE(owner); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); pop_4_error: @@ -11434,8 +11495,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); } @@ -11604,11 +11665,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); UNLOCK_OBJECT(list); // unlock before decrefs! - Py_DECREF(old_value); PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); } DISPATCH(); pop_4_error: @@ -12325,7 +12388,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARA tb = Py_None; } else { + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(tb); + stack_pointer = _PyFrame_GetStackPointer(frame); } assert(PyStackRef_LongCheck(lasti)); (void)lasti; // Shut up compiler warning if asserts are off From 1b6f9697d18380c041cd60868091f8b778ad7d80 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 17 Jan 2025 08:30:55 +0800 Subject: [PATCH 063/110] Address Hugo's review --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 22a8a3760b8035..f89971caa73b30 100644 --- a/configure.ac +++ b/configure.ac @@ -7073,10 +7073,10 @@ AS_VAR_IF( case "$ac_cv_tail_call" in yes*) AC_DEFINE([Py_TAIL_CALL_INTERP], [1], [Define if the C compiler supports efficient proper tail calls.]) - esac + esac ], [] -) +) case $ac_sys_system in From e13aab9e678da17e602d00ee46f1b7612840853a Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 17 Jan 2025 08:36:11 +0800 Subject: [PATCH 064/110] Update 2025-01-10-18-56-20.gh-issue-128563.baDvls.rst --- .../2025-01-10-18-56-20.gh-issue-128563.baDvls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst index 0c73ca294b2cb2..10110fd815f395 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst @@ -1 +1 @@ -A new type of interpreter has been added to CPython. This interpreter uses tail calls for its instruction handlers. Preliminary benchmark results suggest 7-14% geometric mean faster on pyperformance (depending on platform), and up to 45% faster on Python-intensive workloads. This interpreter currently only works on newer compilers, such as ``clang-19``. Other compilers will continue using the old interpreter. Patch by Ken Jin, with ideas by Garret Gu, Haoran Xu, and Josh Haberman. +A new type of interpreter has been added to CPython. This interpreter uses tail calls for its instruction handlers. Preliminary benchmark results suggest 7-11% geometric mean faster on pyperformance (depending on platform), and up to 30% faster on Python-intensive workloads. This interpreter currently only works on newer compilers, such as ``clang-19``. Other compilers will continue using the old interpreter. Patch by Ken Jin, with ideas by Garret Gu, Haoran Xu, and Josh Haberman. From 297f5b9d4ebea376540b9b7a29f8c52cf546e81b Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 17 Jan 2025 08:37:12 +0800 Subject: [PATCH 065/110] Update generated_tail_call_handlers.c.h --- Python/generated_tail_call_handlers.c.h | 119 ++++++++++++++++++------ 1 file changed, 92 insertions(+), 27 deletions(-) diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index b323517226bf98..e8c0fb1fe55be0 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -172,10 +172,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_resume_with_error(TAIL_CALL_PARA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS){ { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP); PREDICTED(BINARY_OP); - _Py_CODEUNIT* const this_instr = next_instr - 2; + _Py_CODEUNIT* const this_instr = next_instr - 6; (void)this_instr; _PyStackRef lhs; _PyStackRef rhs; @@ -200,6 +200,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS){ assert(NB_ADD <= oparg); assert(oparg <= NB_INPLACE_XOR); } + /* Skip 4 cache entries */ // _BINARY_OP { PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); @@ -239,9 +240,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS){ { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; _PyStackRef right; _PyStackRef res; @@ -254,7 +255,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PA DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_ADD_FLOAT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -295,9 +296,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS){ { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; _PyStackRef right; _PyStackRef res; @@ -310,7 +311,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARA DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_ADD_INT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -350,9 +351,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS){ { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; _PyStackRef right; _PyStackRef res; @@ -365,7 +366,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_ DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_ADD_UNICODE { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -402,12 +403,76 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_ TAIL_CALL(resume_with_error); } +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS){ + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_EXTEND); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + /* Skip 1 cache entry */ + // _GUARD_BINARY_OP_EXTEND + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *descr = read_obj(&this_instr[2].cache); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + assert(d && d->guard); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = d->guard(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + DEOPT_IF(!res, BINARY_OP); + } + /* Skip -4 cache entry */ + // _BINARY_OP_EXTEND + { + PyObject *descr = read_obj(&this_instr[2].cache); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = d->action(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS){ { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; _PyStackRef right; // _GUARD_BOTH_UNICODE @@ -419,7 +484,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TA DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_INPLACE_ADD_UNICODE { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -486,9 +551,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS){ { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; _PyStackRef right; _PyStackRef res; @@ -501,7 +566,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CA DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_FLOAT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -542,9 +607,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS){ { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; _PyStackRef right; _PyStackRef res; @@ -557,7 +622,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_INT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -597,9 +662,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS){ { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; _PyStackRef right; _PyStackRef res; @@ -612,7 +677,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CA DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_FLOAT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -653,9 +718,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS){ { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; _PyStackRef right; _PyStackRef res; @@ -668,7 +733,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_INT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -12507,6 +12572,7 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_BINARY_OP_ADD_UNICODE, + [BINARY_OP_EXTEND] = _TAIL_CALL_BINARY_OP_EXTEND, [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE, [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT, [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_BINARY_OP_MULTIPLY_INT, @@ -12753,7 +12819,6 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [146] = _TAIL_CALL_UNKNOWN_OPCODE, [147] = _TAIL_CALL_UNKNOWN_OPCODE, [148] = _TAIL_CALL_UNKNOWN_OPCODE, - [229] = _TAIL_CALL_UNKNOWN_OPCODE, [230] = _TAIL_CALL_UNKNOWN_OPCODE, [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, From e4f214780a6e691dd8a015462a0913e5ea6d1541 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 20 Jan 2025 17:51:16 +0800 Subject: [PATCH 066/110] Make deopts significantly more efficient --- Lib/test/test_generated_cases.py | 2 +- Python/bytecodes.c | 8 +- Python/ceval_macros.h | 14 +- Python/generated_cases.c.h | 458 ++++++++++----------- Python/generated_tail_call_handlers.c.h | 458 ++++++++++----------- Tools/cases_generator/generators_common.py | 2 + 6 files changed, 472 insertions(+), 470 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 1a702f7bf28d61..b1dd600756b6c7 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -460,7 +460,7 @@ def test_predictions(self): INSTRUCTION_STATS(OP3); static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size"); _PyStackRef res; - DEOPT_IF(xxx, OP1); + DEOPT_IF(xxx, OP1, INLINE_CACHE_ENTRIES_OP1); res = Py_None; stack_pointer[-1] = res; DISPATCH(); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ce139c0d5c2b00..72d820d3dfab57 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -45,7 +45,7 @@ #include "ceval_macros.h" /* Flow control macros */ -#define GO_TO_INSTRUCTION(instname) ((void)0) +#define GO_TO_INSTRUCTION(instname, size) ((void)0) #define inst(name, ...) case name: #define op(name, ...) /* NAME is ignored */ @@ -2014,7 +2014,7 @@ dummy_func( // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); } family(LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR) = { @@ -4313,7 +4313,7 @@ dummy_func( frame, this_instr, function, arg); ERROR_IF(err, error); PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(CALL_KW); + GO_TO_INSTRUCTION(CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } op(_MAYBE_EXPAND_METHOD_KW, (callable[1], self_or_null[1], args[oparg], kwnames_in -- func[1], maybe_self[1], args[oparg], kwnames_out)) { @@ -4539,7 +4539,7 @@ dummy_func( _CHECK_PERIODIC; inst(INSTRUMENTED_CALL_FUNCTION_EX, ( -- )) { - GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + GO_TO_INSTRUCTION(CALL_FUNCTION_EX, 0); } op(_MAKE_CALLARGS_A_TUPLE, (func, unused, callargs, kwargs_in if (oparg & 1) -- func, unused, tuple, kwargs_out if (oparg & 1))) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index efadf30157d3b2..259cdd677bb078 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -326,18 +326,18 @@ GETITEM(PyObject *v, Py_ssize_t i) { PyStackRef_XCLOSE(tmp); } while (0) #ifdef Py_TAIL_CALL_INTERP #ifdef LLTRACE -#define GO_TO_INSTRUCTION(op) do { \ +#define GO_TO_INSTRUCTION(op, size) do { \ Py_MUSTTAIL \ - return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - _PyOpcode_Caches[_PyOpcode_Deopt[op]], opcode, oparg, lltrace); \ + return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - size, opcode, oparg, lltrace); \ } while (0) #else -#define GO_TO_INSTRUCTION(op) do { \ +#define GO_TO_INSTRUCTION(op, size) do { \ Py_MUSTTAIL \ - return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - _PyOpcode_Caches[_PyOpcode_Deopt[op]], opcode, oparg); \ + return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - size, opcode, oparg); \ } while (0) #endif #else -#define GO_TO_INSTRUCTION(op) goto PREDICT_ID(op) +#define GO_TO_INSTRUCTION(op, size) goto PREDICT_ID(op) #endif #ifdef Py_STATS @@ -354,12 +354,12 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define UPDATE_MISS_STATS(INSTNAME) ((void)0) #endif -#define DEOPT_IF(COND, INSTNAME) \ +#define DEOPT_IF(COND, INSTNAME, SIZE) \ if ((COND)) { \ /* This is only a single jump on release builds! */ \ UPDATE_MISS_STATS((INSTNAME)); \ assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); \ - GO_TO_INSTRUCTION(INSTNAME); \ + GO_TO_INSTRUCTION(INSTNAME, SIZE); \ } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c7a5223024aad9..4467f06202f3fa 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -73,8 +73,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_ADD_FLOAT @@ -111,8 +111,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_ADD_INT @@ -148,8 +148,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_ADD_UNICODE @@ -193,7 +193,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int res = d->guard(left_o, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); - DEOPT_IF(!res, BINARY_OP); + DEOPT_IF(!res, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip -4 cache entry */ // _BINARY_OP_EXTEND @@ -230,8 +230,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_INPLACE_ADD_UNICODE @@ -248,7 +248,7 @@ next_oparg = CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); - DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP); + DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -294,8 +294,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_FLOAT @@ -332,8 +332,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_INT @@ -369,8 +369,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_FLOAT @@ -407,8 +407,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_INT @@ -540,7 +540,7 @@ dict_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -574,22 +574,22 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); } // _BINARY_SUBSCR_CHECK_FUNC { container = stack_pointer[-2]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR); + DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); assert(PyFunction_Check(getitem_o)); uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR); + DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); } @@ -634,19 +634,19 @@ list_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; #ifdef Py_GIL_DISABLED _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); stack_pointer = _PyFrame_GetStackPointer(frame); - DEOPT_IF(res_o == NULL, BINARY_SUBSCR); + DEOPT_IF(res_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); #else - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); @@ -674,14 +674,14 @@ str_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); @@ -706,12 +706,12 @@ tuple_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); @@ -1101,7 +1101,7 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_AND_ALLOCATE_OBJECT { @@ -1112,17 +1112,17 @@ self = &stack_pointer[-1 - oparg]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(!PyType_Check(callable_o), CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL, INLINE_CACHE_ENTRIES_CALL); assert(tp->tp_new == PyBaseObject_Type.tp_new); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); assert(tp->tp_alloc == PyType_GenericAlloc); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *self_o = PyType_GenericAlloc(tp, 0); @@ -1199,14 +1199,14 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS { null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { @@ -1225,9 +1225,9 @@ callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_EXACT_ARGS { @@ -1236,15 +1236,15 @@ assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_PY_EXACT_ARGS { @@ -1302,7 +1302,7 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_METHOD_VERSION { @@ -1310,11 +1310,11 @@ callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + DEOPT_IF(!PyFunction_Check(func), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); } // _EXPAND_METHOD { @@ -1403,7 +1403,7 @@ self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyType_Check(callable_o), CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; int total_args = oparg; _PyStackRef *arguments = args; @@ -1411,7 +1411,7 @@ arguments--; total_args++; } - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + DEOPT_IF(tp->tp_vectorcall == NULL, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { @@ -1488,8 +1488,8 @@ arguments--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ @@ -1572,8 +1572,8 @@ arguments--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); /* res = func(self, arguments, nargs, kwnames) */ _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1656,11 +1656,11 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL); + DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; @@ -1933,9 +1933,9 @@ arguments--; total_args++; } - DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL); + DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyStackRef cls_stackref = arguments[1]; _PyStackRef inst_stackref = arguments[0]; @@ -2132,7 +2132,7 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + DEOPT_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CHECK_METHOD_VERSION_KW { @@ -2140,11 +2140,11 @@ callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL_KW); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW); + DEOPT_IF(!PyFunction_Check(func), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _EXPAND_METHOD_KW { @@ -2238,8 +2238,8 @@ { callable = &stack_pointer[-3 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL_KW); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW); + DEOPT_IF(PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CALL_KW_NON_PY { @@ -2329,16 +2329,16 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + DEOPT_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CHECK_FUNCTION_VERSION_KW { callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW); + DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL_KW); + DEOPT_IF(func->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _PY_FRAME_KW { @@ -2422,9 +2422,9 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.len, CALL); + DEOPT_IF(callable_o != interp->callable_cache.len, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyStackRef arg_stackref = args[0]; PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); @@ -2465,10 +2465,10 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL); + DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL, INLINE_CACHE_ENTRIES_CALL); assert(self_o != NULL); - DEOPT_IF(!PyList_Check(self_o), CALL); - DEOPT_IF(!LOCK_OBJECT(self_o), CALL); + DEOPT_IF(!PyList_Check(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!LOCK_OBJECT(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); @@ -2511,11 +2511,11 @@ } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -2596,12 +2596,12 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *d_type = method->d_common.d_type; PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -2681,16 +2681,16 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; _PyStackRef self_stackref = args[0]; PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -2753,16 +2753,16 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); + DEOPT_IF(meth->ml_flags != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); _PyStackRef arg_stackref = args[1]; _PyStackRef self_stackref = args[0]; DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type), CALL); + method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -2820,8 +2820,8 @@ { callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL); + DEOPT_IF(PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CALL_NON_PY_GENERAL { @@ -2905,16 +2905,16 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_EXACT_ARGS { @@ -2923,15 +2923,15 @@ assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_PY_EXACT_ARGS { @@ -2986,16 +2986,16 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _PY_FRAME_GENERAL { @@ -3071,8 +3071,8 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Str(arg_o); @@ -3122,8 +3122,8 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PySequence_Tuple(arg_o); @@ -3171,8 +3171,8 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); PyStackRef_CLOSE(arg); @@ -3376,8 +3376,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_FLOAT @@ -3414,16 +3414,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_INT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && _PyLong_DigitCount((PyLongObject *)right_o) <= 1); @@ -3456,8 +3456,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_STR @@ -3540,7 +3540,7 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP); + DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); @@ -3568,7 +3568,7 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP); + DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4075,14 +4075,14 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _FOR_ITER_GEN_FRAME { iter = stack_pointer[-1]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); STAT_INC(FOR_ITER, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, PyStackRef_None); @@ -4123,7 +4123,7 @@ // _ITER_CHECK_LIST { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_LIST { @@ -4175,7 +4175,7 @@ { iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_RANGE { @@ -4217,7 +4217,7 @@ // _ITER_CHECK_TUPLE { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_TUPLE { @@ -4637,7 +4637,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + GO_TO_INSTRUCTION(CALL_FUNCTION_EX, 0); } TARGET(INSTRUMENTED_CALL_KW) { @@ -4661,7 +4661,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(CALL_KW); + GO_TO_INSTRUCTION(CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } TARGET(INSTRUMENTED_END_FOR) { @@ -4851,7 +4851,7 @@ // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); } TARGET(INSTRUMENTED_NOT_TAKEN) { @@ -5394,9 +5394,9 @@ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS @@ -5429,16 +5429,16 @@ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_TYPE_VERSION { uint32_t type_version = read_u32(&this_instr[4].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_CLASS { @@ -5469,17 +5469,17 @@ PyObject *getattribute = read_obj(&this_instr[6].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( @@ -5507,14 +5507,14 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_MANAGED_OBJECT_HAS_VALUES { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { @@ -5522,10 +5522,10 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #ifdef Py_GIL_DISABLED if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } #else attr = PyStackRef_FromPyObjectNew(attr_o); @@ -5557,7 +5557,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_ATTR_METHOD_LAZY_DICT { @@ -5565,7 +5565,7 @@ char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR); + DEOPT_IF(dict != NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 1 cache entry */ // _LOAD_ATTR_METHOD_LAZY_DICT @@ -5600,7 +5600,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_NO_DICT @@ -5636,14 +5636,14 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyDictValues *ivs = _PyObject_InlineValues(owner_o); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_KEYS_VERSION { @@ -5651,7 +5651,7 @@ PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_METHOD_WITH_VALUES { @@ -5686,11 +5686,11 @@ owner = stack_pointer[-1]; uint32_t dict_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR); + DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); mod_keys = keys; } // _LOAD_ATTR_MODULE_FROM_KEYS @@ -5701,11 +5701,11 @@ PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); // Clear mod_keys from stack in case we need to deopt - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); if (!increfed) { - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } #else Py_INCREF(attr_o); @@ -5737,7 +5737,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT @@ -5768,14 +5768,14 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyDictValues *ivs = _PyObject_InlineValues(owner_o); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_KEYS_VERSION { @@ -5783,7 +5783,7 @@ PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { @@ -5808,7 +5808,7 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_TYPE_VERSION { @@ -5816,7 +5816,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_PROPERTY_FRAME @@ -5826,10 +5826,10 @@ assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; PyCodeObject *code = (PyCodeObject *)f->func_code; - DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR); - DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR); - DEOPT_IF(code->co_argcount != 1, LOAD_ATTR); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(code->co_argcount != 1, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); new_frame->localsplus[0] = owner; @@ -5878,7 +5878,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_SLOT { @@ -5886,10 +5886,10 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **addr = (PyObject **)((char *)owner_o + index); PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); - DEOPT_IF(!increfed, LOAD_ATTR); + DEOPT_IF(!increfed, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #else attr = PyStackRef_FromPyObjectNew(attr_o); #endif @@ -5921,14 +5921,14 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_ATTR_WITH_HINT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict_o == NULL, LOAD_ATTR); + DEOPT_IF(dict_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(PyDict_CheckExact((PyObject *)dict_o)); dict = dict_o; } @@ -5937,26 +5937,26 @@ uint16_t hint = read_u16(&this_instr[4].cache); PyObject *attr_o; if (!LOCK_OBJECT(dict)) { - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } if (hint >= (size_t)dict->ma_keys->dk_nentries) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } attr_o = ep->me_value; if (attr_o == NULL) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); @@ -6319,18 +6319,18 @@ { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); assert(DK_IS_UNICODE(keys)); } // _GUARD_BUILTINS_VERSION_PUSH_KEYS { uint16_t version = read_u16(&this_instr[3].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); builtins_keys = keys; assert(DK_IS_UNICODE(builtins_keys)); } @@ -6339,10 +6339,10 @@ uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL); + DEOPT_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -6370,9 +6370,9 @@ { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); globals_keys = keys; assert(DK_IS_UNICODE(globals_keys)); } @@ -6382,10 +6382,10 @@ uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL); + DEOPT_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -6604,8 +6604,8 @@ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6640,8 +6640,8 @@ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -7167,16 +7167,16 @@ INSTRUCTION_STATS(RESUME_CHECK); static_assert(0 == 0, "incorrect cache size"); #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME, 0); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); + DEOPT_IF(eval_breaker != version, RESUME, 0); #ifdef Py_GIL_DISABLED DEOPT_IF(frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME); + ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME, 0); #endif DISPATCH(); } @@ -7346,15 +7346,15 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, SEND); + DEOPT_IF(tstate->interp->eval_frame, SEND, INLINE_CACHE_ENTRIES_SEND); } // _SEND_GEN_FRAME { v = stack_pointer[-1]; receiver = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND, INLINE_CACHE_ENTRIES_SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND, INLINE_CACHE_ENTRIES_SEND); STAT_INC(SEND, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, v); @@ -7544,11 +7544,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); PyTypeObject *tp = Py_TYPE(owner_o); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } } // _GUARD_DORV_NO_DICT @@ -7559,7 +7559,7 @@ if (_PyObject_GetManagedDict(owner_o) || !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } } // _STORE_ATTR_INSTANCE_VALUE @@ -7602,14 +7602,14 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } // _STORE_ATTR_SLOT { value = stack_pointer[-2]; uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; @@ -7639,7 +7639,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } // _STORE_ATTR_WITH_HINT { @@ -7648,12 +7648,12 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, STORE_ATTR); - DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR); + DEOPT_IF(dict == NULL, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); #ifdef Py_GIL_DISABLED if (dict != _PyObject_GetManagedDict(owner_o)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } #endif assert(PyDict_CheckExact((PyObject *)dict)); @@ -7661,17 +7661,17 @@ if (hint >= (size_t)dict->ma_keys->dk_nentries || !DK_IS_UNICODE(dict->ma_keys)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } PyObject *old_value = ep->me_value; if (old_value == NULL) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } _PyFrame_SetStackPointer(frame, stack_pointer); _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); @@ -7907,7 +7907,7 @@ dict_st = stack_pointer[-2]; value = stack_pointer[-3]; PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, @@ -7935,16 +7935,16 @@ value = stack_pointer[-3]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR); + DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); // Ensure index < len(list) if (index >= PyList_GET_SIZE(list)) { UNLOCK_OBJECT(list); - DEOPT_IF(true, STORE_SUBSCR); + DEOPT_IF(true, STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); } STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); @@ -8034,7 +8034,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); } // _REPLACE_WITH_TRUE { @@ -8055,7 +8055,7 @@ /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; - DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL); + DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); DISPATCH(); } @@ -8071,7 +8071,7 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL); + DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value_o)) { assert(_Py_IsImmortal(value_o)); @@ -8096,7 +8096,7 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL); + DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; PyStackRef_CLOSE(value); @@ -8115,7 +8115,7 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL); + DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); res = PyStackRef_False; stack_pointer[-1] = res; @@ -8133,7 +8133,7 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL); + DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); if (value_o == &_Py_STR(empty)) { assert(_Py_IsImmortal(value_o)); @@ -8269,11 +8269,11 @@ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); if (PyList_GET_SIZE(seq_o) != oparg) { UNLOCK_OBJECT(seq_o); - DEOPT_IF(true, UNPACK_SEQUENCE); + DEOPT_IF(true, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq_o); @@ -8298,8 +8298,8 @@ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { @@ -8323,8 +8323,8 @@ seq = stack_pointer[-1]; assert(oparg == 2); PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index e8c0fb1fe55be0..6149f9da769989 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -252,8 +252,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PA left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_ADD_FLOAT @@ -308,8 +308,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARA left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_ADD_INT @@ -363,8 +363,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_ADD_UNICODE @@ -426,7 +426,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAM _PyFrame_SetStackPointer(frame, stack_pointer); int res = d->guard(left_o, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); - DEOPT_IF(!res, BINARY_OP); + DEOPT_IF(!res, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip -4 cache entry */ // _BINARY_OP_EXTEND @@ -481,8 +481,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TA left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_INPLACE_ADD_UNICODE @@ -499,7 +499,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TA next_oparg = CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); - DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP); + DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -563,8 +563,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CA left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_FLOAT @@ -619,8 +619,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_INT @@ -674,8 +674,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CA left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_FLOAT @@ -730,8 +730,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_INT @@ -917,7 +917,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PAR dict_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -969,22 +969,22 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); } // _BINARY_SUBSCR_CHECK_FUNC { container = stack_pointer[-2]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR); + DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); assert(PyFunction_Check(getitem_o)); uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR); + DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); } @@ -1047,19 +1047,19 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL list_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; #ifdef Py_GIL_DISABLED _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); stack_pointer = _PyFrame_GetStackPointer(frame); - DEOPT_IF(res_o == NULL, BINARY_SUBSCR); + DEOPT_IF(res_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); #else - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); @@ -1105,14 +1105,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_ str_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); @@ -1155,12 +1155,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CAL tuple_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); @@ -1712,7 +1712,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_C /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_AND_ALLOCATE_OBJECT { @@ -1723,17 +1723,17 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_C self = &stack_pointer[-1 - oparg]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(!PyType_Check(callable_o), CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL, INLINE_CACHE_ENTRIES_CALL); assert(tp->tp_new == PyBaseObject_Type.tp_new); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); assert(tp->tp_alloc == PyType_GenericAlloc); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *self_o = PyType_GenericAlloc(tp, 0); @@ -1828,14 +1828,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS { null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { @@ -1854,9 +1854,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_EXACT_ARGS { @@ -1865,15 +1865,15 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_PY_EXACT_ARGS { @@ -1949,7 +1949,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_C /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_METHOD_VERSION { @@ -1957,11 +1957,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_C callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + DEOPT_IF(!PyFunction_Check(func), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); } // _EXPAND_METHOD { @@ -2068,7 +2068,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyType_Check(callable_o), CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; int total_args = oparg; _PyStackRef *arguments = args; @@ -2076,7 +2076,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR arguments--; total_args++; } - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + DEOPT_IF(tp->tp_vectorcall == NULL, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { @@ -2171,8 +2171,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARA arguments--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ @@ -2273,8 +2273,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS( arguments--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); /* res = func(self, arguments, nargs, kwnames) */ _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2375,11 +2375,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS) args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL); + DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; @@ -2724,9 +2724,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS arguments--; total_args++; } - DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL); + DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyStackRef cls_stackref = arguments[1]; _PyStackRef inst_stackref = arguments[0]; @@ -2959,7 +2959,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_P /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + DEOPT_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CHECK_METHOD_VERSION_KW { @@ -2967,11 +2967,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_P callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL_KW); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW); + DEOPT_IF(!PyFunction_Check(func), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _EXPAND_METHOD_KW { @@ -3083,8 +3083,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) { callable = &stack_pointer[-3 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL_KW); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW); + DEOPT_IF(PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CALL_KW_NON_PY { @@ -3192,16 +3192,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + DEOPT_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CHECK_FUNCTION_VERSION_KW { callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW); + DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL_KW); + DEOPT_IF(func->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _PY_FRAME_KW { @@ -3303,9 +3303,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.len, CALL); + DEOPT_IF(callable_o != interp->callable_cache.len, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyStackRef arg_stackref = args[0]; PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); @@ -3364,10 +3364,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAM PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL); + DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL, INLINE_CACHE_ENTRIES_CALL); assert(self_o != NULL); - DEOPT_IF(!PyList_Check(self_o), CALL); - DEOPT_IF(!LOCK_OBJECT(self_o), CALL); + DEOPT_IF(!PyList_Check(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!LOCK_OBJECT(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); @@ -3428,11 +3428,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -3531,12 +3531,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *d_type = method->d_common.d_type; PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -3634,16 +3634,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TA args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; _PyStackRef self_stackref = args[0]; PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -3724,16 +3724,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CA total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); + DEOPT_IF(meth->ml_flags != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); _PyStackRef arg_stackref = args[1]; _PyStackRef self_stackref = args[0]; DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type), CALL); + method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -3809,8 +3809,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PA { callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL); + DEOPT_IF(PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CALL_NON_PY_GENERAL { @@ -3912,16 +3912,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PAR /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_EXACT_ARGS { @@ -3930,15 +3930,15 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PAR assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_PY_EXACT_ARGS { @@ -4011,16 +4011,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _PY_FRAME_GENERAL { @@ -4114,8 +4114,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS){ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Str(arg_o); @@ -4183,8 +4183,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PySequence_Tuple(arg_o); @@ -4250,8 +4250,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS){ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); PyStackRef_CLOSE(arg); @@ -4545,8 +4545,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAM left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_FLOAT @@ -4601,16 +4601,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS) left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_INT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && _PyLong_DigitCount((PyLongObject *)right_o) <= 1); @@ -4661,8 +4661,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS) left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_STR @@ -4781,7 +4781,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAM left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP); + DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); @@ -4827,7 +4827,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP); + DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5712,14 +5712,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS){ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _FOR_ITER_GEN_FRAME { iter = stack_pointer[-1]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); STAT_INC(FOR_ITER, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, PyStackRef_None); @@ -5778,7 +5778,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS){ // _ITER_CHECK_LIST { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_LIST { @@ -5848,7 +5848,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS) { iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_RANGE { @@ -5908,7 +5908,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS) // _ITER_CHECK_TUPLE { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_TUPLE { @@ -6508,7 +6508,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TA frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + GO_TO_INSTRUCTION(CALL_FUNCTION_EX, 0); } pop_4_error: TAIL_CALL(pop_4_error); @@ -6550,7 +6550,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_P stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(CALL_KW); + GO_TO_INSTRUCTION(CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } pop_4_error: TAIL_CALL(pop_4_error); @@ -6884,7 +6884,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAI // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); } pop_4_error: TAIL_CALL(pop_4_error); @@ -7751,9 +7751,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS @@ -7804,16 +7804,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_C owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_TYPE_VERSION { uint32_t type_version = read_u32(&this_instr[4].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_CLASS { @@ -7862,17 +7862,17 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDE PyObject *getattribute = read_obj(&this_instr[6].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( @@ -7918,14 +7918,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_MANAGED_OBJECT_HAS_VALUES { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { @@ -7933,10 +7933,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #ifdef Py_GIL_DISABLED if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } #else attr = PyStackRef_FromPyObjectNew(attr_o); @@ -7986,7 +7986,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_ATTR_METHOD_LAZY_DICT { @@ -7994,7 +7994,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_ char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR); + DEOPT_IF(dict != NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 1 cache entry */ // _LOAD_ATTR_METHOD_LAZY_DICT @@ -8047,7 +8047,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CA uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_NO_DICT @@ -8101,14 +8101,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAI uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyDictValues *ivs = _PyObject_InlineValues(owner_o); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_KEYS_VERSION { @@ -8116,7 +8116,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAI PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_METHOD_WITH_VALUES { @@ -8169,11 +8169,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM owner = stack_pointer[-1]; uint32_t dict_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR); + DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); mod_keys = keys; } // _LOAD_ATTR_MODULE_FROM_KEYS @@ -8184,11 +8184,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); // Clear mod_keys from stack in case we need to deopt - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); if (!increfed) { - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } #else Py_INCREF(attr_o); @@ -8238,7 +8238,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT( uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT @@ -8287,14 +8287,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VAL uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyDictValues *ivs = _PyObject_InlineValues(owner_o); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_KEYS_VERSION { @@ -8302,7 +8302,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VAL PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { @@ -8345,7 +8345,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_TYPE_VERSION { @@ -8353,7 +8353,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_PROPERTY_FRAME @@ -8363,10 +8363,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; PyCodeObject *code = (PyCodeObject *)f->func_code; - DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR); - DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR); - DEOPT_IF(code->co_argcount != 1, LOAD_ATTR); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(code->co_argcount != 1, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); new_frame->localsplus[0] = owner; @@ -8433,7 +8433,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_SLOT { @@ -8441,10 +8441,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **addr = (PyObject **)((char *)owner_o + index); PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); - DEOPT_IF(!increfed, LOAD_ATTR); + DEOPT_IF(!increfed, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #else attr = PyStackRef_FromPyObjectNew(attr_o); #endif @@ -8494,14 +8494,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_ATTR_WITH_HINT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict_o == NULL, LOAD_ATTR); + DEOPT_IF(dict_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(PyDict_CheckExact((PyObject *)dict_o)); dict = dict_o; } @@ -8510,26 +8510,26 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA uint16_t hint = read_u16(&this_instr[4].cache); PyObject *attr_o; if (!LOCK_OBJECT(dict)) { - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } if (hint >= (size_t)dict->ma_keys->dk_nentries) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } attr_o = ep->me_value; if (attr_o == NULL) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); @@ -9144,18 +9144,18 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); assert(DK_IS_UNICODE(keys)); } // _GUARD_BUILTINS_VERSION_PUSH_KEYS { uint16_t version = read_u16(&this_instr[3].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); builtins_keys = keys; assert(DK_IS_UNICODE(builtins_keys)); } @@ -9164,10 +9164,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL); + DEOPT_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -9213,9 +9213,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PAR { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); globals_keys = keys; assert(DK_IS_UNICODE(globals_keys)); } @@ -9225,10 +9225,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PAR uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL); + DEOPT_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -9555,8 +9555,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_P PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9609,8 +9609,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -10550,16 +10550,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS){ INSTRUCTION_STATS(RESUME_CHECK); static_assert(0 == 0, "incorrect cache size"); #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME, 0); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); + DEOPT_IF(eval_breaker != version, RESUME, 0); #ifdef Py_GIL_DISABLED DEOPT_IF(frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME); + ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME, 0); #endif } DISPATCH(); @@ -10801,15 +10801,15 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS){ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, SEND); + DEOPT_IF(tstate->interp->eval_frame, SEND, INLINE_CACHE_ENTRIES_SEND); } // _SEND_GEN_FRAME { v = stack_pointer[-1]; receiver = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND, INLINE_CACHE_ENTRIES_SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND, INLINE_CACHE_ENTRIES_SEND); STAT_INC(SEND, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, v); @@ -11107,11 +11107,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_C uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); PyTypeObject *tp = Py_TYPE(owner_o); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } } // _GUARD_DORV_NO_DICT @@ -11122,7 +11122,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_C if (_PyObject_GetManagedDict(owner_o) || !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } } // _STORE_ATTR_INSTANCE_VALUE @@ -11183,14 +11183,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } // _STORE_ATTR_SLOT { value = stack_pointer[-2]; uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; @@ -11238,7 +11238,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } // _STORE_ATTR_WITH_HINT { @@ -11247,12 +11247,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, STORE_ATTR); - DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR); + DEOPT_IF(dict == NULL, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); #ifdef Py_GIL_DISABLED if (dict != _PyObject_GetManagedDict(owner_o)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } #endif assert(PyDict_CheckExact((PyObject *)dict)); @@ -11260,17 +11260,17 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P if (hint >= (size_t)dict->ma_keys->dk_nentries || !DK_IS_UNICODE(dict->ma_keys)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } PyObject *old_value = ep->me_value; if (old_value == NULL) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } _PyFrame_SetStackPointer(frame, stack_pointer); _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); @@ -11668,7 +11668,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARA dict_st = stack_pointer[-2]; value = stack_pointer[-3]; PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, @@ -11714,16 +11714,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_ value = stack_pointer[-3]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR); + DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); // Ensure index < len(list) if (index >= PyList_GET_SIZE(list)) { UNLOCK_OBJECT(list); - DEOPT_IF(true, STORE_SUBSCR); + DEOPT_IF(true, STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); } STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); @@ -11867,7 +11867,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PA uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); } // _REPLACE_WITH_TRUE { @@ -11906,7 +11906,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS){ /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; - DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL); + DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); } DISPATCH(); @@ -11940,7 +11940,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS){ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL); + DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value_o)) { assert(_Py_IsImmortal(value_o)); @@ -11983,7 +11983,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS){ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL); + DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; PyStackRef_CLOSE(value); @@ -12020,7 +12020,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS){ /* Skip 2 cache entries */ value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL); + DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); res = PyStackRef_False; stack_pointer[-1] = res; @@ -12056,7 +12056,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS){ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL); + DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); if (value_o == &_Py_STR(empty)) { assert(_Py_IsImmortal(value_o)); @@ -12300,11 +12300,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_P seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); if (PyList_GET_SIZE(seq_o) != oparg) { UNLOCK_OBJECT(seq_o); - DEOPT_IF(true, UNPACK_SEQUENCE); + DEOPT_IF(true, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq_o); @@ -12347,8 +12347,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { @@ -12390,8 +12390,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_C seq = stack_pointer[-1]; assert(oparg == 2); PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index f54afbb880d2fa..1b9f09800d09c7 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -160,6 +160,8 @@ def deopt_if( assert inst is not None assert inst.family is not None self.out.emit(inst.family.name) + self.out.emit(", ") + self.out.emit(inst.family.size) self.out.emit(");\n") return not always_true(first_tkn) From aa3d408e1bd35ef5599d86a26a9ed7a2f036b881 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 20 Jan 2025 17:55:12 +0800 Subject: [PATCH 067/110] reduce diff --- Python/bytecodes.c | 4 ---- Python/generated_cases.c.h | 2 -- Python/generated_tail_call_handlers.c.h | 2 -- 3 files changed, 8 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 72d820d3dfab57..2adc94233c06a2 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1368,7 +1368,6 @@ dummy_func( Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); - INPUTS_DEAD(); goto exception_unwind; } } @@ -1389,9 +1388,6 @@ dummy_func( else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); monitor_reraise(tstate, frame, this_instr); - INPUTS_DEAD(); - none = PyStackRef_NULL; - value = PyStackRef_NULL; goto exception_unwind; } } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 4467f06202f3fa..55ac76bc02d313 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3290,8 +3290,6 @@ _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - none = PyStackRef_NULL; - value = PyStackRef_NULL; goto exception_unwind; } stack_pointer[-3] = none; diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 6149f9da769989..c98fda6eaa7bb4 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -4423,8 +4423,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS){ _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - none = PyStackRef_NULL; - value = PyStackRef_NULL; goto exception_unwind; } stack_pointer[-3] = none; From aa1a5f6138298273cb78da77a10b55c5751cf2ce Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 22 Jan 2025 11:03:14 +0800 Subject: [PATCH 068/110] Update generated_tail_call_handlers.c.h --- Python/generated_tail_call_handlers.c.h | 879 ++++++++++++++---------- 1 file changed, 509 insertions(+), 370 deletions(-) diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index c98fda6eaa7bb4..359d6ced23e458 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -115,7 +115,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAM } /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { + if (frame->lltrace >= 5) { lltrace_resume_frame(frame); } #endif @@ -174,7 +174,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP); - PREDICTED(BINARY_OP); + PREDICTED_BINARY_OP; _Py_CODEUNIT* const this_instr = next_instr - 6; (void)this_instr; _PyStackRef lhs; @@ -843,7 +843,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR); - PREDICTED(BINARY_SUBSCR); + PREDICTED_BINARY_SUBSCR; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef container; @@ -1356,28 +1356,30 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SLICE); - _PyStackRef start; - _PyStackRef stop; - _PyStackRef step = PyStackRef_NULL; + _PyStackRef *args; _PyStackRef slice; - if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } - stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; - start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; + args = &stack_pointer[-oparg]; + assert(oparg == 2 || oparg == 3); + _PyStackRef start = args[0]; + _PyStackRef stop = args[1]; PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); - PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); + PyObject * step_o = NULL; + if (oparg == 3) { + step_o = PyStackRef_AsPyObjectBorrow(args[2]); + } PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); - PyStackRef_CLOSE(start); - PyStackRef_CLOSE(stop); - PyStackRef_XCLOSE(step); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (slice_o == NULL) { - stack_pointer += -2 - ((oparg == 3) ? 1 : 0); + stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); goto error; } slice = PyStackRef_FromPyObjectSteal(slice_o); - stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; - stack_pointer += -1 - ((oparg == 3) ? 1 : 0); + stack_pointer[-oparg] = slice; + stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -1522,7 +1524,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL); - PREDICTED(CALL); + PREDICTED_CALL; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; _PyStackRef *callable; @@ -2442,23 +2444,23 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_FUNCTION_EX); - PREDICTED(CALL_FUNCTION_EX); + PREDICTED_CALL_FUNCTION_EX; _Py_CODEUNIT* const this_instr = next_instr - 1; (void)this_instr; _PyStackRef func; _PyStackRef callargs; - _PyStackRef kwargs_in = PyStackRef_NULL; + _PyStackRef kwargs_in; _PyStackRef tuple; - _PyStackRef kwargs_out = PyStackRef_NULL; + _PyStackRef kwargs_out; _PyStackRef func_st; _PyStackRef callargs_st; - _PyStackRef kwargs_st = PyStackRef_NULL; + _PyStackRef kwargs_st; _PyStackRef result; // _MAKE_CALLARGS_A_TUPLE { - if (oparg & 1) { kwargs_in = stack_pointer[-(oparg & 1)]; } - callargs = stack_pointer[-1 - (oparg & 1)]; - func = stack_pointer[-3 - (oparg & 1)]; + kwargs_in = stack_pointer[-1]; + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (PyTuple_CheckExact(callargs_o)) { tuple = callargs; @@ -2500,8 +2502,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM assert(PyTuple_CheckExact(callargs)); PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - stack_pointer[-1 - (oparg & 1)] = callargs_st; - if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; + stack_pointer[-2] = callargs_st; + stack_pointer[-1] = kwargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, @@ -2544,7 +2546,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -2 - (oparg & 1); + stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( @@ -2565,8 +2567,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM assert(PyTuple_CheckExact(callargs)); PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[-1 - (oparg & 1)] = callargs_st; - if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; + stack_pointer[-2] = callargs_st; + stack_pointer[-1] = kwargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); result_o = PyObject_Call(func, callargs, kwargs); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2576,11 +2578,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(callargs_st); PyStackRef_CLOSE(func_st); - if (result_o == NULL) { - stack_pointer += -3 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + if (result_o == NULL) goto pop_4_error; result = PyStackRef_FromPyObjectSteal(result_o); } // _CHECK_PERIODIC @@ -2588,19 +2586,19 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-3 - (oparg & 1)] = result; - stack_pointer += -2 - (oparg & 1); + stack_pointer[-4] = result; + stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) goto error; - stack_pointer += 2 + (oparg & 1); + stack_pointer += 3; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-3 - (oparg & 1)] = result; - stack_pointer += -2 - (oparg & 1); + stack_pointer[-4] = result; + stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -2771,7 +2769,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW); - PREDICTED(CALL_KW); + PREDICTED_CALL_KW; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; _PyStackRef *callable; @@ -4454,7 +4452,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP); - PREDICTED(COMPARE_OP); + PREDICTED_COMPARE_OP; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef left; @@ -4706,7 +4704,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP); - PREDICTED(CONTAINS_OP); + PREDICTED_CONTAINS_OP; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef left; @@ -5623,7 +5621,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER); - PREDICTED(FOR_ITER); + PREDICTED_FOR_ITER; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef iter; @@ -6506,7 +6504,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TA frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - GO_TO_INSTRUCTION(CALL_FUNCTION_EX, 0); + + goto PREDICTED_CALL_FUNCTION_EX; } pop_4_error: TAIL_CALL(pop_4_error); @@ -6548,7 +6547,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_P stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + goto PREDICTED_CALL_KW; } pop_4_error: TAIL_CALL(pop_4_error); @@ -6828,8 +6827,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARA if (tstate->tracing) { PyCodeObject *code = _PyFrame_GetCode(frame); _PyFrame_SetStackPointer(frame, stack_pointer); - original_opcode = code->_co_monitoring->lines[(int)(this_instr - _PyFrame_GetBytecode(frame))].original_opcode; + int index = (int)(this_instr - _PyFrame_GetBytecode(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); + original_opcode = code->_co_monitoring->lines->data[index*code->_co_monitoring->lines->bytes_per_entry]; next_instr = this_instr; } else { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6882,7 +6882,37 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAI // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + goto PREDICTED_LOAD_SUPER_ATTR; + } + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_METHOD(TAIL_CALL_PARAMS){ + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_METHOD); + /* Skip 1 cache entry */ + // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we + // don't want to specialize instrumented instructions + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + goto PREDICTED_LOAD_SUPER_METHOD; } pop_4_error: TAIL_CALL(pop_4_error); @@ -7214,9 +7244,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_C // _RETURN_VALUE { retval = val; - #if TIER_ONE - assert(!frame->is_entry_frame); - #endif + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = retval; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -7286,9 +7314,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CA // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. - #if TIER_ONE - assert(!frame->is_entry_frame); - #endif + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); frame->instr_ptr++; PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); @@ -7349,7 +7375,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAM INSTRUCTION_STATS(INTERPRETER_EXIT); _PyStackRef retval; retval = stack_pointer[-1]; - assert(frame->is_entry_frame); + assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); assert(_PyFrame_IsIncomplete(frame)); /* Restore previous frame and return. */ tstate->current_frame = frame->previous; @@ -7643,12 +7669,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR); - PREDICTED(LOAD_ATTR); + PREDICTED_LOAD_ATTR; _Py_CODEUNIT* const this_instr = next_instr - 10; (void)this_instr; _PyStackRef owner; _PyStackRef attr; - _PyStackRef self_or_null = PyStackRef_NULL; // _SPECIALIZE_LOAD_ATTR { owner = stack_pointer[-1]; @@ -7656,7 +7681,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ (void)counter; #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadAttr(owner, next_instr, name); @@ -7670,50 +7695,15 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ /* Skip 8 cache entries */ // _LOAD_ATTR { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - PyObject *attr_o; - if (oparg & 1) { - /* Designed to work in tandem with CALL, pushes two values. */ - attr_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (is_meth) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - meth | self | arg1 | ... | argN - */ - assert(attr_o != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership - } - else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - meth | NULL | arg1 | ... | argN - */ - PyStackRef_CLOSE(owner); - if (attr_o == NULL) goto pop_1_error; - self_or_null = PyStackRef_NULL; - } - } - else { - /* Classic, pushes one value. */ - _PyFrame_SetStackPointer(frame, stack_pointer); - attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(owner); - if (attr_o == NULL) goto pop_1_error; - /* We need to define self_or_null on all paths */ - self_or_null = PyStackRef_NULL; - } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(owner); + if (attr_o == NULL) goto pop_1_error; attr = PyStackRef_FromPyObjectSteal(attr_o); } stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = self_or_null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); pop_4_error: @@ -7742,7 +7732,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _CHECK_ATTR_CLASS { @@ -7760,13 +7749,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); pop_4_error: @@ -7795,7 +7780,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_C static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _CHECK_ATTR_CLASS { @@ -7819,13 +7803,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_C STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); pop_4_error: @@ -7859,7 +7839,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDE uint32_t func_version = read_u32(&this_instr[4].cache); PyObject *getattribute = read_obj(&this_instr[6].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); @@ -7872,7 +7851,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDE assert(code->co_argcount == 2); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( tstate, PyStackRef_FromPyObjectNew(f), 2, frame); // Manipulate stack directly because we exit with DISPATCH_INLINED(). @@ -7908,7 +7887,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { @@ -7940,197 +7918,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS){ - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - } - // _CHECK_ATTR_METHOD_LAZY_DICT - { - uint16_t dictoffset = read_u16(&this_instr[4].cache); - char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); - /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - } - /* Skip 1 cache entry */ - // _LOAD_ATTR_METHOD_LAZY_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS){ - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_METHOD_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS){ - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyDictValues *ivs = _PyObject_InlineValues(owner_o); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - } - // _LOAD_ATTR_METHOD_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - /* Cached method object */ - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); pop_4_error: @@ -8160,7 +7951,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM _PyStackRef owner; PyDictKeysObject *mod_keys; _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _CHECK_ATTR_MODULE_PUSH_KEYS { @@ -8193,14 +7983,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM attr = PyStackRef_FromPyObjectSteal(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); pop_4_error: @@ -8242,7 +8028,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT( // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT { PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -8305,7 +8090,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VAL // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); PyStackRef_CLOSE(owner); @@ -8357,7 +8141,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR // _LOAD_ATTR_PROPERTY_FRAME { PyObject *fget = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; PyCodeObject *code = (PyCodeObject *)f->func_code; @@ -8423,7 +8206,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { @@ -8447,14 +8229,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); pop_4_error: @@ -8484,7 +8262,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA _PyStackRef owner; PyDictObject *dict; _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { @@ -8514,7 +8291,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA UNLOCK_OBJECT(dict); DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); @@ -8532,14 +8309,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); UNLOCK_OBJECT(dict); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); pop_4_error: @@ -8647,7 +8420,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); - PREDICTED(LOAD_CONST); + PREDICTED_LOAD_CONST; _Py_CODEUNIT* const this_instr = next_instr - 1; (void)this_instr; _PyStackRef value; @@ -9070,18 +8843,17 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL); - PREDICTED(LOAD_GLOBAL); + PREDICTED_LOAD_GLOBAL; _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; _PyStackRef *res; - _PyStackRef null = PyStackRef_NULL; // _SPECIALIZE_LOAD_GLOBAL { uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); @@ -9098,15 +8870,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS){ // _LOAD_GLOBAL { res = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*res)) goto error; - null = PyStackRef_NULL; } - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -9136,7 +8906,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyDictKeysObject *builtins_keys; _PyStackRef res; - _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_GLOBALS_VERSION { @@ -9171,11 +8940,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA res = PyStackRef_FromPyObjectSteal(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); - null = PyStackRef_NULL; } stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -9205,7 +8972,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PAR static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyDictKeysObject *globals_keys; _PyStackRef res; - _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_GLOBALS_VERSION_PUSH_KEYS { @@ -9232,11 +8998,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PAR res = PyStackRef_FromPyObjectSteal(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); - null = PyStackRef_NULL; } stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -9296,7 +9060,271 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS){ TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_METHOD(TAIL_CALL_PARAMS){ + { + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_METHOD); + PREDICTED_LOAD_METHOD; + _Py_CODEUNIT* const this_instr = next_instr - 10; + (void)this_instr; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self_or_null; + // _SPECIALIZE_LOAD_METHOD + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadMethod(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_METHOD); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 8 cache entries */ + // _LOAD_METHOD + { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *attr_o; + /* Designed to work in tandem with CALL, pushes two values. */ + attr_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (is_meth) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr_o != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + } + else { + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + meth | NULL | arg1 | ... | argN + */ + PyStackRef_CLOSE(owner); + if (attr_o == NULL) goto pop_1_error; + self_or_null = PyStackRef_NULL; + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + } + stack_pointer[-1] = attr; + stack_pointer[0] = self_or_null; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_METHOD_LAZY_DICT(TAIL_CALL_PARAMS){ + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_METHOD_LAZY_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); + } + // _CHECK_ATTR_METHOD_LAZY_DICT + { + uint16_t dictoffset = read_u16(&this_instr[4].cache); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); + /* This object has a __dict__, just not yet created */ + DEOPT_IF(dict != NULL, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); + } + /* Skip 1 cache entry */ + // _LOAD_METHOD_LAZY_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_METHOD_NO_DICT(TAIL_CALL_PARAMS){ + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_METHOD_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_METHOD_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_METHOD_WITH_VALUES(TAIL_CALL_PARAMS){ + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_METHOD_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); + } + // _LOAD_METHOD_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + /* Cached method object */ + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS){ { frame->instr_ptr = next_instr; next_instr += 1; @@ -9423,14 +9451,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR); - PREDICTED(LOAD_SUPER_ATTR); + PREDICTED_LOAD_SUPER_ATTR; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef global_super_st; _PyStackRef class_st; _PyStackRef self_st; _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; // _SPECIALIZE_LOAD_SUPER_ATTR { class_st = stack_pointer[-2]; @@ -9438,11 +9465,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION_FT - int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 0); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } @@ -9456,7 +9482,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + if (opcode >= MIN_INSTRUMENTED_OPCODE) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( @@ -9476,7 +9502,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + if (opcode >= MIN_INSTRUMENTED_OPCODE) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9509,11 +9535,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) goto error; attr = PyStackRef_FromPyObjectSteal(attr_o); - null = PyStackRef_NULL; } stack_pointer[0] = attr; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -9588,11 +9612,130 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_P TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_METHOD(TAIL_CALL_PARAMS){ + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_METHOD); + PREDICTED_LOAD_SUPER_METHOD; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef null; + // _SPECIALIZE_LOAD_SUPER_METHOD + { + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 1); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_SUPER_METHOD); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _LOAD_SUPER_ATTR + { + self_st = stack_pointer[-1]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode >= MIN_INSTRUMENTED_OPCODE) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + goto pop_3_error; + } + } + // we make no attempt to optimize here; specializations should + // handle any case whose performance we care about + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (opcode >= MIN_INSTRUMENTED_OPCODE) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(super); + } + } + } + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + if (super == NULL) goto pop_3_error; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + Py_DECREF(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) goto error; + attr = PyStackRef_FromPyObjectSteal(attr_o); + } + // _PUSH_NULL + { + null = PyStackRef_NULL; + } + stack_pointer[0] = attr; + stack_pointer[1] = null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_METHOD_METHOD(TAIL_CALL_PARAMS){ { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + INSTRUCTION_STATS(LOAD_SUPER_METHOD_METHOD); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); _PyStackRef global_super_st; _PyStackRef class_st; @@ -9607,8 +9750,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_METHOD, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_METHOD, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -10297,9 +10440,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_NULL); - _PyStackRef res; - res = PyStackRef_NULL; - stack_pointer[0] = res; + _PyStackRef null; + null = PyStackRef_NULL; + stack_pointer[0] = null; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } @@ -10458,7 +10601,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME); - PREDICTED(RESUME); + PREDICTED_RESUME; _Py_CODEUNIT* const this_instr = next_instr - 1; (void)this_instr; // _LOAD_BYTECODE @@ -10638,9 +10781,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS){ _PyStackRef retval; _PyStackRef res; retval = stack_pointer[-1]; - #if TIER_ONE - assert(!frame->is_entry_frame); - #endif + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = retval; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -10683,7 +10824,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND); - PREDICTED(SEND); + PREDICTED_SEND; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef receiver; @@ -10711,7 +10852,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS){ v = stack_pointer[-1]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); PyObject *retval_o; - assert(!frame->is_entry_frame); + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); if ((tstate->interp->eval_frame == NULL) && (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) @@ -11032,7 +11173,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR); - PREDICTED(STORE_ATTR); + PREDICTED_STORE_ATTR; _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; _PyStackRef owner; @@ -11594,7 +11735,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR); - PREDICTED(STORE_SUBSCR); + PREDICTED_STORE_SUBSCR; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef container; @@ -11796,7 +11937,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL); - PREDICTED(TO_BOOL); + PREDICTED_TO_BOOL; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; _PyStackRef value; @@ -12230,7 +12371,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE); - PREDICTED(UNPACK_SEQUENCE); + PREDICTED_UNPACK_SEQUENCE; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef seq; @@ -12499,9 +12640,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS){ // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. - #if TIER_ONE - assert(!frame->is_entry_frame); - #endif + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); frame->instr_ptr++; PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); @@ -12671,6 +12810,7 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD, [INSTRUMENTED_LINE] = _TAIL_CALL_INSTRUMENTED_LINE, [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR, + [INSTRUMENTED_LOAD_SUPER_METHOD] = _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_METHOD, [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_INSTRUMENTED_NOT_TAKEN, [INSTRUMENTED_POP_ITER] = _TAIL_CALL_INSTRUMENTED_POP_ITER, [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE, @@ -12692,9 +12832,6 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE, - [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT, - [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT, - [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES, [LOAD_ATTR_MODULE] = _TAIL_CALL_LOAD_ATTR_MODULE, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, @@ -12717,12 +12854,17 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_LOAD_GLOBAL_BUILTIN, [LOAD_GLOBAL_MODULE] = _TAIL_CALL_LOAD_GLOBAL_MODULE, [LOAD_LOCALS] = _TAIL_CALL_LOAD_LOCALS, + [LOAD_METHOD] = _TAIL_CALL_LOAD_METHOD, + [LOAD_METHOD_LAZY_DICT] = _TAIL_CALL_LOAD_METHOD_LAZY_DICT, + [LOAD_METHOD_NO_DICT] = _TAIL_CALL_LOAD_METHOD_NO_DICT, + [LOAD_METHOD_WITH_VALUES] = _TAIL_CALL_LOAD_METHOD_WITH_VALUES, [LOAD_NAME] = _TAIL_CALL_LOAD_NAME, [LOAD_SMALL_INT] = _TAIL_CALL_LOAD_SMALL_INT, [LOAD_SPECIAL] = _TAIL_CALL_LOAD_SPECIAL, [LOAD_SUPER_ATTR] = _TAIL_CALL_LOAD_SUPER_ATTR, [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_LOAD_SUPER_ATTR_ATTR, - [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_LOAD_SUPER_ATTR_METHOD, + [LOAD_SUPER_METHOD] = _TAIL_CALL_LOAD_SUPER_METHOD, + [LOAD_SUPER_METHOD_METHOD] = _TAIL_CALL_LOAD_SUPER_METHOD_METHOD, [MAKE_CELL] = _TAIL_CALL_MAKE_CELL, [MAKE_FUNCTION] = _TAIL_CALL_MAKE_FUNCTION, [MAP_ADD] = _TAIL_CALL_MAP_ADD, @@ -12786,8 +12928,6 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, - [118] = _TAIL_CALL_UNKNOWN_OPCODE, - [119] = _TAIL_CALL_UNKNOWN_OPCODE, [120] = _TAIL_CALL_UNKNOWN_OPCODE, [121] = _TAIL_CALL_UNKNOWN_OPCODE, [122] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -12821,7 +12961,6 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, [233] = _TAIL_CALL_UNKNOWN_OPCODE, - [234] = _TAIL_CALL_UNKNOWN_OPCODE, }; #undef TIER_ONE #undef IN_TAIL_CALL_INTERP From fda19f46a3fee2d9053dfa63924f2489c65bc0cf Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 22 Jan 2025 11:28:11 +0800 Subject: [PATCH 069/110] partial fix (waiting for #129112 ) --- Include/internal/pycore_frame.h | 4 -- Python/ceval_macros.h | 15 +++--- Python/generated_cases.c.h | 40 ++++++++-------- Python/generated_tail_call_handlers.c.h | 48 +++++++++---------- Tools/cases_generator/tier1_generator.py | 2 +- .../tier1_tail_call_generator.py | 48 +++++++++++++++---- 6 files changed, 92 insertions(+), 65 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index c2eb4393ba150f..14dc91e54298ce 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -220,8 +220,6 @@ _PyFrame_Initialize( frame->localsplus[i] = PyStackRef_NULL; } - frame->is_entry_frame = 0; - #ifdef Py_GIL_DISABLED // On GIL disabled, we walk the entire stack in GC. Since stacktop // is not always in sync with the real stack pointer, we have @@ -402,8 +400,6 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int frame->visited = 0; frame->return_offset = 0; - frame->is_entry_frame = 0; - #ifdef Py_GIL_DISABLED assert(code->co_nlocalsplus == 0); for (int i = 0; i < code->co_stacksize; i++) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index ac60d6d1918d8e..35852015103691 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -307,13 +307,14 @@ GETITEM(PyObject *v, Py_ssize_t i) { #endif #ifdef Py_TAIL_CALL_INTERP -if ((COND)) { \ - /* This is only a single jump on release builds! */ \ - UPDATE_MISS_STATS((INSTNAME)); \ - assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); \ - Py_MUSTTAIL \ - return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - size, opcode, oparg); \ -} +# define DEOPT_IF(COND, INSTNAME, SIZE) \ + if ((COND)) { \ + /* This is only a single jump on release builds! */ \ + UPDATE_MISS_STATS((INSTNAME)); \ + assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); \ + Py_MUSTTAIL \ + return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - size, opcode, oparg); \ + } #else # define DEOPT_IF(COND, INSTNAME, SIZE) \ if ((COND)) { \ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 74be7a406b5cb0..f2186c918ee50b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -13,7 +13,7 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP); - PREDICTED_BINARY_OP; + PREDICTED_BINARY_OP:; _Py_CODEUNIT* const this_instr = next_instr - 6; (void)this_instr; _PyStackRef lhs; @@ -484,7 +484,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR); - PREDICTED_BINARY_SUBSCR; + PREDICTED_BINARY_SUBSCR:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef container; @@ -931,7 +931,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL); - PREDICTED_CALL; + PREDICTED_CALL:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; _PyStackRef *callable; @@ -1707,7 +1707,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_FUNCTION_EX); - PREDICTED_CALL_FUNCTION_EX; + PREDICTED_CALL_FUNCTION_EX:; _Py_CODEUNIT* const this_instr = next_instr - 1; (void)this_instr; _PyStackRef func; @@ -1960,7 +1960,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW); - PREDICTED_CALL_KW; + PREDICTED_CALL_KW:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; _PyStackRef *callable; @@ -3301,7 +3301,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP); - PREDICTED_COMPARE_OP; + PREDICTED_COMPARE_OP:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef left; @@ -3481,7 +3481,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP); - PREDICTED_CONTAINS_OP; + PREDICTED_CONTAINS_OP:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef left; @@ -4002,7 +4002,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER); - PREDICTED_FOR_ITER; + PREDICTED_FOR_ITER:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef iter; @@ -5312,7 +5312,7 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR); - PREDICTED_LOAD_ATTR; + PREDICTED_LOAD_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 10; (void)this_instr; _PyStackRef owner; @@ -5829,7 +5829,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); - PREDICTED_LOAD_CONST; + PREDICTED_LOAD_CONST:; _Py_CODEUNIT* const this_instr = next_instr - 1; (void)this_instr; _PyStackRef value; @@ -6072,7 +6072,7 @@ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL); - PREDICTED_LOAD_GLOBAL; + PREDICTED_LOAD_GLOBAL:; _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; _PyStackRef *res; @@ -6222,7 +6222,7 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_METHOD); - PREDICTED_LOAD_METHOD; + PREDICTED_LOAD_METHOD:; _Py_CODEUNIT* const this_instr = next_instr - 10; (void)this_instr; _PyStackRef owner; @@ -6482,7 +6482,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR); - PREDICTED_LOAD_SUPER_ATTR; + PREDICTED_LOAD_SUPER_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef global_super_st; @@ -6612,7 +6612,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_METHOD); - PREDICTED_LOAD_SUPER_METHOD; + PREDICTED_LOAD_SUPER_METHOD:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef global_super_st; @@ -7182,7 +7182,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME); - PREDICTED_RESUME; + PREDICTED_RESUME:; _Py_CODEUNIT* const this_instr = next_instr - 1; (void)this_instr; // _LOAD_BYTECODE @@ -7333,7 +7333,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND); - PREDICTED_SEND; + PREDICTED_SEND:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef receiver; @@ -7574,7 +7574,7 @@ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR); - PREDICTED_STORE_ATTR; + PREDICTED_STORE_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; _PyStackRef owner; @@ -7938,7 +7938,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR); - PREDICTED_STORE_SUBSCR; + PREDICTED_STORE_SUBSCR:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef container; @@ -8068,7 +8068,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL); - PREDICTED_TO_BOOL; + PREDICTED_TO_BOOL:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; _PyStackRef value; @@ -8304,7 +8304,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE); - PREDICTED_UNPACK_SEQUENCE; + PREDICTED_UNPACK_SEQUENCE:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef seq; diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 359d6ced23e458..6469b8507617e2 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -174,7 +174,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP); - PREDICTED_BINARY_OP; + PREDICTED_BINARY_OP:; _Py_CODEUNIT* const this_instr = next_instr - 6; (void)this_instr; _PyStackRef lhs; @@ -843,7 +843,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR); - PREDICTED_BINARY_SUBSCR; + PREDICTED_BINARY_SUBSCR:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef container; @@ -1524,7 +1524,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL); - PREDICTED_CALL; + PREDICTED_CALL:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; _PyStackRef *callable; @@ -2444,7 +2444,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_FUNCTION_EX); - PREDICTED_CALL_FUNCTION_EX; + PREDICTED_CALL_FUNCTION_EX:; _Py_CODEUNIT* const this_instr = next_instr - 1; (void)this_instr; _PyStackRef func; @@ -2769,7 +2769,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW); - PREDICTED_CALL_KW; + PREDICTED_CALL_KW:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; _PyStackRef *callable; @@ -4452,7 +4452,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP); - PREDICTED_COMPARE_OP; + PREDICTED_COMPARE_OP:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef left; @@ -4704,7 +4704,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP); - PREDICTED_CONTAINS_OP; + PREDICTED_CONTAINS_OP:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef left; @@ -5621,7 +5621,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER); - PREDICTED_FOR_ITER; + PREDICTED_FOR_ITER:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef iter; @@ -6505,7 +6505,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TA next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - goto PREDICTED_CALL_FUNCTION_EX; + Py_MUSTTAIL return (INSTRUCTION_TABLE[CALL_FUNCTION_EX])(frame, stack_pointer, tstate, next_instr - 1 - 0, opcode, oparg); } pop_4_error: TAIL_CALL(pop_4_error); @@ -6547,7 +6547,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_P stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - goto PREDICTED_CALL_KW; + Py_MUSTTAIL return (INSTRUCTION_TABLE[CALL_KW])(frame, stack_pointer, tstate, next_instr - 1 - INLINE_CACHE_ENTRIES_CALL_KW, opcode, oparg); } pop_4_error: TAIL_CALL(pop_4_error); @@ -6882,7 +6882,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAI // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - goto PREDICTED_LOAD_SUPER_ATTR; + Py_MUSTTAIL return (INSTRUCTION_TABLE[LOAD_SUPER_ATTR])(frame, stack_pointer, tstate, next_instr - 1 - INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR, opcode, oparg); } pop_4_error: TAIL_CALL(pop_4_error); @@ -6912,7 +6912,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_METHOD(T // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - goto PREDICTED_LOAD_SUPER_METHOD; + Py_MUSTTAIL return (INSTRUCTION_TABLE[LOAD_SUPER_METHOD])(frame, stack_pointer, tstate, next_instr - 1 - INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR, opcode, oparg); } pop_4_error: TAIL_CALL(pop_4_error); @@ -7669,7 +7669,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR); - PREDICTED_LOAD_ATTR; + PREDICTED_LOAD_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 10; (void)this_instr; _PyStackRef owner; @@ -8420,7 +8420,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); - PREDICTED_LOAD_CONST; + PREDICTED_LOAD_CONST:; _Py_CODEUNIT* const this_instr = next_instr - 1; (void)this_instr; _PyStackRef value; @@ -8843,7 +8843,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL); - PREDICTED_LOAD_GLOBAL; + PREDICTED_LOAD_GLOBAL:; _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; _PyStackRef *res; @@ -9065,7 +9065,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_METHOD(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_METHOD); - PREDICTED_LOAD_METHOD; + PREDICTED_LOAD_METHOD:; _Py_CODEUNIT* const this_instr = next_instr - 10; (void)this_instr; _PyStackRef owner; @@ -9451,7 +9451,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR); - PREDICTED_LOAD_SUPER_ATTR; + PREDICTED_LOAD_SUPER_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef global_super_st; @@ -9617,7 +9617,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_METHOD(TAIL_CALL_PARA frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_METHOD); - PREDICTED_LOAD_SUPER_METHOD; + PREDICTED_LOAD_SUPER_METHOD:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef global_super_st; @@ -10601,7 +10601,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME); - PREDICTED_RESUME; + PREDICTED_RESUME:; _Py_CODEUNIT* const this_instr = next_instr - 1; (void)this_instr; // _LOAD_BYTECODE @@ -10824,7 +10824,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND); - PREDICTED_SEND; + PREDICTED_SEND:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef receiver; @@ -11173,7 +11173,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR); - PREDICTED_STORE_ATTR; + PREDICTED_STORE_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; _PyStackRef owner; @@ -11735,7 +11735,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR); - PREDICTED_STORE_SUBSCR; + PREDICTED_STORE_SUBSCR:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef container; @@ -11937,7 +11937,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL); - PREDICTED_TO_BOOL; + PREDICTED_TO_BOOL:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; _PyStackRef value; @@ -12371,7 +12371,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE); - PREDICTED_UNPACK_SEQUENCE; + PREDICTED_UNPACK_SEQUENCE:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef seq; diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 1af78c6ebf845f..69ac74700ac4be 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -141,7 +141,7 @@ def write_single_inst(out: CWriter, emitter: Emitter, name: str, inst: Instructi out.emit(f"next_instr += {inst.size};\n") out.emit(f"INSTRUCTION_STATS({name});\n") if inst.is_target: - out.emit(f"PREDICTED_{name};\n") + out.emit(f"PREDICTED_{name}:;\n") if needs_this: out.emit(f"_Py_CODEUNIT* const this_instr = next_instr - {inst.size};\n") out.emit(unused_guard) diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index 2f0da243f97da9..f87c70212048ea 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -13,17 +13,24 @@ write_header, CWriter, Emitter, + TokenIterator, ) from analyzer import ( Analysis, + Instruction, analyze_files, + Uop, ) from tier1_generator import ( write_single_inst ) +from lexer import Token + +from stack import Storage + DEFAULT_INPUT = ROOT / "Python/bytecodes.c" DEFAULT_OUTPUT = ROOT / "Python/generated_tail_call_handlers.c.h" @@ -33,6 +40,37 @@ TARGET_LABEL = "TAIL_CALL_TARGET" +class TailCallEmitter(Emitter): + + def __init__(self, out: CWriter, analysis: Analysis): + super().__init__(out) + self.analysis = analysis + + def go_to_instruction( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + next(tkn_iter) + name = next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + assert name.kind == "IDENTIFIER" + self.emit("\n") + inst = self.analysis.instructions[name.text] + fam = None + # Search for the family (if any) + for family_name, family in self.analysis.families.items(): + if inst.name == family_name: + fam = family + break + size = fam.size if fam is not None else 0 + self.emit(f"Py_MUSTTAIL return (INSTRUCTION_TABLE[{name.text}])(frame, stack_pointer, tstate, next_instr - 1 - {size}, opcode, oparg);\n") + return True + def generate_label_handlers(infile: TextIO, outfile: TextIO) -> list[str]: out = CWriter(outfile, 0, False) str_in = infile.read() @@ -64,14 +102,6 @@ def generate_label_handlers(infile: TextIO, outfile: TextIO) -> list[str]: out.emit("\n") return function_protos -# For unit testing. -def generate_label_handlers_from_files( - infilename: str, outfilename: str -) -> None: - with open(infilename, "r") as infile, open(outfilename, "w") as outfile: - generate_label_handlers(infile, outfile) - - def function_proto(name: str) -> str: return f"Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_{name}(TAIL_CALL_PARAMS)" @@ -96,7 +126,7 @@ def generate_tier1( with open(DEFAULT_CEVAL_INPUT, "r") as infile: err_labels = generate_label_handlers(infile, outfile) - emitter = Emitter(out) + emitter = TailCallEmitter(out, analysis) out.emit("\n") for name, inst in sorted(analysis.instructions.items()): out.emit("\n") From 1a216072f8fa94dd241f7f76f88e5ee3cf31899e Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 23 Jan 2025 13:09:17 +0800 Subject: [PATCH 070/110] Make it opt-in --- configure | 38 -------------------------------------- configure.ac | 25 ------------------------- 2 files changed, 63 deletions(-) diff --git a/configure b/configure index aa35e7e18ac294..85603ffbd2b167 100755 --- a/configure +++ b/configure @@ -29281,44 +29281,6 @@ esac fi - -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether $CC supports efficient proper tail calls" >&5 -printf %s "checking whether $CC supports efficient proper tail calls... " >&6; } -if test ${ac_cv_tail_call+y} -then : - printf %s "(cached) " >&6 -else case e in #( - e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - - -#if __has_attribute(musttail) -# if __has_attribute(preserve_none) -// Only allowlist x86_64 and AArch64. Add more architectures as compilers gain support for them. -# if defined(__x86_64__) || defined(__aarch64__) -# define PROPER_TAIL_CALLS 1 -# endif -# endif -#endif - -#ifndef PROPER_TAIL_CALLS -# error "No proper tail calls" -#endif - - -_ACEOF -if ac_fn_c_try_compile "$LINENO" -then : - ac_cv_tail_call=yes -else case e in #( - e) ac_cv_tail_call=no ;; -esac -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; -esac -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_tail_call" >&5 -printf "%s\n" "$ac_cv_tail_call" >&6; } # Do not enable tail-calling interpreter if tier 2 is enabled. if ${tier2_flags:+false} : then : diff --git a/configure.ac b/configure.ac index 052ca8845c0c45..ff7544dfb460c4 100644 --- a/configure.ac +++ b/configure.ac @@ -7040,31 +7040,6 @@ fi ], [AC_MSG_RESULT([no value specified])]) - -AC_CACHE_CHECK([whether $CC supports efficient proper tail calls], [ac_cv_tail_call], -AC_COMPILE_IFELSE([AC_LANG_SOURCE([[[ - -#if __has_attribute(musttail) -# if __has_attribute(preserve_none) -// Only allowlist x86_64 and AArch64. Add more architectures as compilers gain support for them. -# if defined(__x86_64__) || defined(__aarch64__) -# define PROPER_TAIL_CALLS 1 -# endif -# endif -#endif - -#ifndef PROPER_TAIL_CALLS -# error "No proper tail calls" -#endif - -]]])], -[ac_cv_tail_call=yes], -[ac_cv_tail_call=no], -[if test "${with_tail_call_interp+set}" = set; then - ac_cv_tail_call="$with_tail_call_interp -- configured --with(out)-tail-call-interp" - else - ac_cv_tail_call=no - fi])) # Do not enable tail-calling interpreter if tier 2 is enabled. AS_VAR_IF( [tier2_flags], From 15da9c7a4280287df4624f51b83043c57619c82e Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 25 Jan 2025 11:23:19 +0800 Subject: [PATCH 071/110] Squashed commit of the following: commit f89f147940645823bda29669b7870adb67530cdc Author: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat Jan 25 11:18:43 2025 +0800 Address review commit e4a9de79140b7f5df66db581a9579616dbe942d5 Author: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu Jan 23 02:05:13 2025 +0800 Regen files commit 5d561305ee0db2c72e581b95751e80b2bd23c2bb Author: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu Jan 23 02:00:44 2025 +0800 Address review by removing in test cases generator commit e1f94758cdec9d071e3951c5dad1b711e9bdd02e Author: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed Jan 22 09:39:54 2025 +0800 Remove entry_frame commit 15ca6dd3927e599f884c7c4528a7bf280cea632f Author: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed Jan 22 09:27:37 2025 +0800 fix upstream changes commit fdd45409d80d29e1933e3530b1b9559f1437c199 Merge: 1ec80334516 86c1a60d5a2 Author: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed Jan 22 09:27:31 2025 +0800 Merge remote-tracking branch 'upstream/main' into labels-in-dsl commit 1ec803345165dc6b917e4ec66b90da364e9efa87 Author: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue Jan 21 20:16:42 2025 +0800 Lint commit c6df7a1285e76c0c92fdefb88478f24b2c00c90a Author: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue Jan 21 20:14:10 2025 +0800 cleanup diff commit 63a88ab5677d70f47392b3868d148c2765e85c37 Author: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue Jan 21 20:13:12 2025 +0800 remove outdated file commit b911bb1bb60577e9708d3a92e9f17298d5f7d11b Author: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue Jan 21 20:11:27 2025 +0800 Move labels into tier 1 generator commit af4bf1a34d389bff47761eec7d2dd7f61973b421 Author: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue Jan 21 15:52:53 2025 +0800 Add to generated files commit 3ae88a5b8863063b84fcf0de2cc131b10822d415 Author: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue Jan 21 09:01:39 2025 +0800 lint commit 5cccb986ba393cf38643b7ccb0b2d5cc28567188 Author: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue Jan 21 08:58:53 2025 +0800 Fix mypy commit 20825373364ceec32fd3a115e3814f3d08420b16 Author: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue Jan 21 08:54:50 2025 +0800 Move labels in ceval.c to bytecodes.c --- Lib/test/test_generated_cases.py | 67 +++++++++- Python/bytecodes.c | 120 +++++++++++++++++ Python/ceval.c | 145 --------------------- Python/generated_cases.c.h | 156 +++++++++++++++++++++++ Tools/cases_generator/analyzer.py | 21 ++- Tools/cases_generator/lexer.py | 3 + Tools/cases_generator/parser.py | 1 + Tools/cases_generator/parsing.py | 19 ++- Tools/cases_generator/tier1_generator.py | 58 ++++++++- 9 files changed, 435 insertions(+), 155 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 5fcc7bac9c39c5..c5532547339f47 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -250,12 +250,12 @@ def run_cases_test(self, input: str, expected: str): ) with open(self.temp_output_filename) as temp_output: - lines = temp_output.readlines() - while lines and lines[0].startswith(("// ", "#", " #", "\n")): - lines.pop(0) - while lines and lines[-1].startswith(("#", "\n")): - lines.pop(-1) - actual = "".join(lines) + lines = temp_output.read() + _, rest = lines.split(tier1_generator.INSTRUCTION_START_MARKER) + instructions, labels_with_prelude_and_postlude = rest.split(tier1_generator.INSTRUCTION_END_MARKER) + _, labels_with_postlude = labels_with_prelude_and_postlude.split(tier1_generator.LABEL_START_MARKER) + labels, _ = labels_with_postlude.split(tier1_generator.LABEL_END_MARKER) + actual = instructions + labels # if actual.strip() != expected.strip(): # print("Actual:") # print(actual) @@ -1641,6 +1641,61 @@ def test_kill_in_wrong_order(self): with self.assertRaises(SyntaxError): self.run_cases_test(input, "") + def test_complex_label(self): + input = """ + label(my_label) { + // Comment + do_thing() + if (complex) { + goto other_label; + } + goto other_label2; + } + """ + + output = """ + my_label: + { + // Comment + do_thing() + if (complex) { + goto other_label; + } + goto other_label2; + } + """ + self.run_cases_test(input, output) + + def test_multiple_labels(self): + input = """ + label(my_label_1) { + // Comment + do_thing1(); + goto my_label_2; + } + + label(my_label_2) { + // Comment + do_thing2(); + goto my_label_3; + } + """ + + output = """ + my_label_1: + { + // Comment + do_thing1(); + goto my_label_2; + } + + my_label_2: + { + // Comment + do_thing2(); + goto my_label_3; + } + """ class TestGeneratedTailCallErorHandlers(unittest.TestCase): def setUp(self) -> None: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 2317f426a774c3..f7c768f6ca5157 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -53,6 +53,7 @@ #define super(name) static int SUPER_##name #define family(name, ...) static int family_##name #define pseudo(name) static int pseudo_##name +#define label(name) name: /* Annotations */ #define guard @@ -5193,6 +5194,125 @@ dummy_func( assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); } + label(pop_4_error) { + STACK_SHRINK(1); + goto pop_3_error; + } + + label(pop_3_error) { + STACK_SHRINK(1); + goto pop_2_error; + } + + label(pop_2_error) { + STACK_SHRINK(1); + goto pop_1_error; + } + + label(pop_1_error) { + STACK_SHRINK(1); + goto error; + } + + label(error) { + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); + goto exception_unwind; + } + + label(exception_unwind) { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (frame->lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + + label(exit_unwind) { + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == &entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + goto resume_with_error; + } + + label(resume_with_error) { + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } // END BYTECODES // } diff --git a/Python/ceval.c b/Python/ceval.c index efaf5bafdd7945..33c24ccd3c8916 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -919,154 +919,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DISPATCH(); #endif -#ifndef Py_TAIL_CALL_INTERP - /* Start instructions */ -#if !USE_COMPUTED_GOTOS - dispatch_opcode: - switch (opcode) -#endif - { #include "generated_cases.c.h" -#if USE_COMPUTED_GOTOS - _unknown_opcode: -#else - EXTRA_CASES // From pycore_opcode_metadata.h, a 'case' for each unused opcode -#endif - /* Tell C compilers not to hold the opcode variable in the loop. - next_instr points the current instruction without TARGET(). */ - opcode = next_instr->op.code; - _PyErr_Format(tstate, PyExc_SystemError, - "%U:%d: unknown opcode %d", - _PyFrame_GetCode(frame)->co_filename, - PyUnstable_InterpreterFrame_GetLine(frame), - opcode); - goto error; - - } /* End instructions */ - - /* This should never be reached. Every opcode should end with DISPATCH() - or goto error. */ - Py_UNREACHABLE(); -#endif -TAIL_CALL_TARGET(pop_4_error): - STACK_SHRINK(1); -TAIL_CALL_TARGET(pop_3_error): - STACK_SHRINK(1); -TAIL_CALL_TARGET(pop_2_error): - STACK_SHRINK(1); -TAIL_CALL_TARGET(pop_1_error): - STACK_SHRINK(1); -TAIL_CALL_TARGET(error): - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(!frame->is_entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -TAIL_CALL_TARGET(exception_unwind): - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (frame->lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - -#ifdef Py_TAIL_CALL_INTERP -# ifdef IN_TAIL_CALL_INTERP - DISPATCH(); -# else -# ifdef LLTRACE - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0, lltrace); -# else - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0); -# endif -# endif -#else - DISPATCH(); -#endif - } - -TAIL_CALL_TARGET(exit_unwind): - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(!frame->is_entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame->is_entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -TAIL_CALL_TARGET(resume_with_error): - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - -/* END_BASE_INTERPRETER */ #ifdef _Py_TIER2 // Tier 2 is also here! diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f2186c918ee50b..4f4a858ce2c804 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,6 +8,13 @@ #endif #define TIER_ONE 1 +#if !USE_COMPUTED_GOTOS + dispatch_opcode: + switch (opcode) +#endif + { + /* BEGIN INSTRUCTIONS */ + TARGET(BINARY_OP) { frame->instr_ptr = next_instr; @@ -8518,4 +8525,153 @@ assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } + + /* END INSTRUCTIONS */ +#if USE_COMPUTED_GOTOS + _unknown_opcode: +#else + EXTRA_CASES // From pycore_opcode_metadata.h, a 'case' for each unused opcode +#endif + /* Tell C compilers not to hold the opcode variable in the loop. + next_instr points the current instruction without TARGET(). */ + opcode = next_instr->op.code; + _PyErr_Format(tstate, PyExc_SystemError, + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); + goto error; + + } + + /* This should never be reached. Every opcode should end with DISPATCH() + or goto error. */ + Py_UNREACHABLE(); + /* BEGIN LABELS */ + + pop_4_error: + { + STACK_SHRINK(1); + goto pop_3_error; + } + + pop_3_error: + { + STACK_SHRINK(1); + goto pop_2_error; + } + + pop_2_error: + { + STACK_SHRINK(1); + goto pop_1_error; + } + + pop_1_error: + { + STACK_SHRINK(1); + goto error; + } + + error: + { + /* Double-check exception status. */ + #ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } + #else + assert(_PyErr_Occurred(tstate)); + #endif + + /* Log traceback info. */ + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); + goto exception_unwind; + } + + exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ + #ifdef LLTRACE + if (frame->lltrace >= 5) { + lltrace_resume_frame(frame); + } + #endif + DISPATCH(); + } + + exit_unwind: + { + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == &entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + goto resume_with_error; + } + + resume_with_error: + { + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + +/* END LABELS */ #undef TIER_ONE diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 1f15c3bb9c88af..8a1b2f07d96a75 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -255,6 +255,12 @@ def is_super(self) -> bool: return False +@dataclass +class Label: + name: str + body: list[lexer.Token] + + @dataclass class PseudoInstruction: name: str @@ -288,6 +294,7 @@ class Analysis: uops: dict[str, Uop] families: dict[str, Family] pseudos: dict[str, PseudoInstruction] + labels: dict[str, Label] opmap: dict[str, int] have_arg: int min_instrumented: int @@ -949,6 +956,13 @@ def add_pseudo( ) +def add_label( + label: parser.LabelDef, + labels: dict[str, Label], +) -> None: + labels[label.name] = Label(label.name, label.block.tokens) + + def assign_opcodes( instructions: dict[str, Instruction], families: dict[str, Family], @@ -1067,6 +1081,7 @@ def analyze_forest(forest: list[parser.AstNode]) -> Analysis: uops: dict[str, Uop] = {} families: dict[str, Family] = {} pseudos: dict[str, PseudoInstruction] = {} + labels: dict[str, Label] = {} for node in forest: match node: case parser.InstDef(name): @@ -1081,6 +1096,8 @@ def analyze_forest(forest: list[parser.AstNode]) -> Analysis: pass case parser.Pseudo(): pass + case parser.LabelDef(): + pass case _: assert False for node in forest: @@ -1092,6 +1109,8 @@ def analyze_forest(forest: list[parser.AstNode]) -> Analysis: add_family(node, instructions, families) case parser.Pseudo(): add_pseudo(node, instructions, pseudos) + case parser.LabelDef(): + add_label(node, labels) case _: pass for uop in uops.values(): @@ -1117,7 +1136,7 @@ def analyze_forest(forest: list[parser.AstNode]) -> Analysis: families["BINARY_OP"].members.append(inst) opmap, first_arg, min_instrumented = assign_opcodes(instructions, families, pseudos) return Analysis( - instructions, uops, families, pseudos, opmap, first_arg, min_instrumented + instructions, uops, families, pseudos, labels, opmap, first_arg, min_instrumented ) diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index 303a1c02705a3b..473414e6070356 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -213,6 +213,9 @@ def choice(*opts: str) -> str: # A macro in the DSL MACRO = "MACRO" kwds.append(MACRO) +# A label in the DSL +LABEL = "LABEL" +kwds.append(LABEL) keywords = {name.lower(): name for name in kwds} ANNOTATION = "ANNOTATION" diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index db672ad5501f15..68bbb88719e682 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -3,6 +3,7 @@ Macro, Pseudo, Family, + LabelDef, Parser, Context, CacheEffect, diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index b50bb627b6c41c..151421b27983a5 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -149,8 +149,13 @@ class Pseudo(Node): targets: list[str] # opcodes this can be replaced by as_sequence: bool +@dataclass +class LabelDef(Node): + name: str + block: Block -AstNode = InstDef | Macro | Pseudo | Family + +AstNode = InstDef | Macro | Pseudo | Family | LabelDef class Parser(PLexer): @@ -164,6 +169,18 @@ def definition(self) -> AstNode | None: return pseudo if inst := self.inst_def(): return inst + if label := self.label_def(): + return label + return None + + @contextual + def label_def(self) -> LabelDef | None: + if self.expect(lx.LABEL): + if self.expect(lx.LPAREN): + if tkn := self.expect(lx.IDENTIFIER): + if self.expect(lx.RPAREN): + if block := self.block(): + return LabelDef(tkn.text, block) return None @contextual diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 69ac74700ac4be..d5fbaca75e5cc6 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -32,6 +32,10 @@ FOOTER = "#undef TIER_ONE\n" +INSTRUCTION_START_MARKER = "/* BEGIN INSTRUCTIONS */" +INSTRUCTION_END_MARKER = "/* END INSTRUCTIONS */" +LABEL_START_MARKER = "/* BEGIN LABELS */" +LABEL_END_MARKER = "/* END LABELS */" def declare_variable(var: StackItem, out: CWriter) -> None: @@ -166,13 +170,64 @@ def generate_tier1( ) -> None: write_header(__file__, filenames, outfile) outfile.write( - """ + f""" #ifdef TIER_TWO #error "This file is for Tier 1 only" #endif #define TIER_ONE 1 + +#if !USE_COMPUTED_GOTOS + dispatch_opcode: + switch (opcode) +#endif + {{ + {INSTRUCTION_START_MARKER} """ ) + generate_tier1_cases(analysis, outfile, lines) + outfile.write(f""" + {INSTRUCTION_END_MARKER} +#if USE_COMPUTED_GOTOS + _unknown_opcode: +#else + EXTRA_CASES // From pycore_opcode_metadata.h, a 'case' for each unused opcode +#endif + /* Tell C compilers not to hold the opcode variable in the loop. + next_instr points the current instruction without TARGET(). */ + opcode = next_instr->op.code; + _PyErr_Format(tstate, PyExc_SystemError, + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); + goto error; + + }} + + /* This should never be reached. Every opcode should end with DISPATCH() + or goto error. */ + Py_UNREACHABLE(); + {LABEL_START_MARKER} +""") + generate_tier1_labels(analysis, outfile, lines) + outfile.write(f"{LABEL_END_MARKER}\n") + outfile.write(FOOTER) + +def generate_tier1_labels( + analysis: Analysis, outfile: TextIO, lines: bool +) -> None: + out = CWriter(outfile, 2, lines) + out.emit("\n") + for name, label in analysis.labels.items(): + out.emit(f"{name}:\n") + for tkn in label.body: + out.emit(tkn) + out.emit("\n") + out.emit("\n") + +def generate_tier1_cases( + analysis: Analysis, outfile: TextIO, lines: bool +) -> None: out = CWriter(outfile, 2, lines) emitter = Emitter(out) out.emit("\n") @@ -185,7 +240,6 @@ def generate_tier1( out.start_line() out.emit("}") out.emit("\n") - outfile.write(FOOTER) arg_parser = argparse.ArgumentParser( From 665c0f6a26d170f2742cec1e44b5606905e3909d Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 25 Jan 2025 15:38:01 +0800 Subject: [PATCH 072/110] Catch up with labels-as-dsl branch --- Lib/test/test_generated_cases.py | 149 ----------------- Python/bytecodes.c | 9 +- Python/ceval.c | 7 - Python/ceval_macros.h | 2 +- Python/generated_cases.c.h | 12 +- Python/generated_tail_call_handlers.c.h | 158 ++++++++---------- Tools/cases_generator/generators_common.py | 13 ++ Tools/cases_generator/tier1_generator.py | 3 + .../tier1_tail_call_generator.py | 103 ++++++++---- 9 files changed, 174 insertions(+), 282 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index c5532547339f47..f53fd302bca3a0 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1697,155 +1697,6 @@ def test_multiple_labels(self): } """ -class TestGeneratedTailCallErorHandlers(unittest.TestCase): - def setUp(self) -> None: - super().setUp() - self.maxDiff = None - - self.temp_dir = tempfile.gettempdir() - self.temp_input_filename = os.path.join(self.temp_dir, "input.txt") - self.temp_output_filename = os.path.join(self.temp_dir, "output.txt") - self.temp_metadata_filename = os.path.join(self.temp_dir, "metadata.txt") - self.temp_pymetadata_filename = os.path.join(self.temp_dir, "pymetadata.txt") - self.temp_executor_filename = os.path.join(self.temp_dir, "executor.txt") - - def tearDown(self) -> None: - for filename in [ - self.temp_input_filename, - self.temp_output_filename, - self.temp_metadata_filename, - self.temp_pymetadata_filename, - self.temp_executor_filename, - ]: - try: - os.remove(filename) - except: - pass - super().tearDown() - - def run_cases_test(self, input: str, expected: str): - with open(self.temp_input_filename, "w+") as temp_input: - temp_input.write(textwrap.dedent(input)) - temp_input.flush() - - with handle_stderr(): - tier1_tail_call_generator.generate_label_handlers_from_files( - self.temp_input_filename, self.temp_output_filename - ) - - with open(self.temp_output_filename) as temp_output: - lines = temp_output.readlines() - while lines and lines[0].startswith(("// ", "#", " #", "\n")): - lines.pop(0) - while lines and lines[-1].startswith(("#", "\n")): - lines.pop(-1) - actual = "".join(lines) - - self.assertEqual(actual.strip(), textwrap.dedent(expected).strip()) - - def test_correctly_finds_pyeval_framedefault(self): - input = """ - PyObject* _Py_HOT_FUNCTION - _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) - { - - /* END_BASE_INTERPRETER */ - } - """ - output = """ - """ - self.run_cases_test(input, output) - - def test_simple_label(self): - input = """ - PyObject* _Py_HOT_FUNCTION - _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) - { - - TAIL_CALL_TARGET(error): - DO_THING(); - /* END_BASE_INTERPRETER */ - } - """ - output = """ - Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); - - Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS) - { - DO_THING(); - /* END_BASE_INTERPRETER */ - } - """ - self.run_cases_test(input, output) - - def test_fallthrough_label(self): - input = """ - PyObject* _Py_HOT_FUNCTION - _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) - { - - TAIL_CALL_TARGET(error): - DO_THING(); - TAIL_CALL_TARGET(fallthrough): - DO_THING2(); - /* END_BASE_INTERPRETER */ - } - """ - output = """ - Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); - Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_fallthrough(TAIL_CALL_PARAMS); - - Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS) - { - DO_THING(); - TAIL_CALL(fallthrough); - } - - Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_fallthrough(TAIL_CALL_PARAMS) - { - DO_THING2(); - /* END_BASE_INTERPRETER */ - } - """ - self.run_cases_test(input, output) - - def test_transform_gotos(self): - input = """ - PyObject* _Py_HOT_FUNCTION - _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) - { - - TAIL_CALL_TARGET(error): - if (thing) { - goto fallthrough; - } - DO_THING(); - TAIL_CALL_TARGET(fallthrough): - DO_THING2(); - /* END_BASE_INTERPRETER */ - } - """ - output = """ - Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); - Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_fallthrough(TAIL_CALL_PARAMS); - - Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS) - { - if (thing) { - TAIL_CALL(fallthrough); - } - DO_THING(); - TAIL_CALL(fallthrough); - } - - Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_fallthrough(TAIL_CALL_PARAMS) - { - DO_THING2(); - /* END_BASE_INTERPRETER */ - } - """ - self.run_cases_test(input, output) - class TestGeneratedAbstractCases(unittest.TestCase): def setUp(self) -> None: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f7c768f6ca5157..807d20f27f1abd 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5287,7 +5287,14 @@ dummy_func( lltrace_resume_frame(frame); } #endif + // This is a little complicated... + // If we are in a tail call handler, we want to tail call (DISPATCH). + // If we're not then we need the shim frame. +#if defined(Py_TAIL_CALL_INTERP) && !defined(IN_TAIL_CALL_INTERP) + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0); +#else DISPATCH(); +#endif } label(exit_unwind) { @@ -5299,7 +5306,7 @@ dummy_func( frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; - if (frame == &entry_frame) { + if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { /* Restore previous frame and exit */ tstate->current_frame = frame->previous; tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; diff --git a/Python/ceval.c b/Python/ceval.c index 33c24ccd3c8916..86d69ffc39d37a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -808,7 +808,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #ifdef Py_STATS int lastopcode = 0; #endif -#ifndef Py_TAIL_CALL_INTERP uint8_t opcode; /* Current opcode */ int oparg; /* Current opcode argument, if any */ @@ -835,7 +834,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif /* Push frame */ entry_frame.previous = tstate->current_frame; - entry_frame.is_entry_frame = 1; frame->previous = &entry_frame; tstate->current_frame = frame; @@ -910,18 +908,13 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif #ifdef Py_TAIL_CALL_INTERP -#ifdef LLTRACE - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0, lltrace); -#else return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0); -#endif #else DISPATCH(); #endif #include "generated_cases.c.h" - #ifdef _Py_TIER2 // Tier 2 is also here! diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 35852015103691..83718f64025884 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -313,7 +313,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { UPDATE_MISS_STATS((INSTNAME)); \ assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); \ Py_MUSTTAIL \ - return (INSTRUCTION_TABLE[op])(frame, stack_pointer, tstate, next_instr - 1 - size, opcode, oparg); \ + return (INSTRUCTION_TABLE[INSTNAME])(frame, stack_pointer, tstate, next_instr - 1 - SIZE, opcode, oparg); \ } #else # define DEOPT_IF(COND, INSTNAME, SIZE) \ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 4f4a858ce2c804..d443c615e33520 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,6 +8,8 @@ #endif #define TIER_ONE 1 +#ifndef Py_TAIL_CALL_INTERP + #if !USE_COMPUTED_GOTOS dispatch_opcode: switch (opcode) @@ -8547,6 +8549,7 @@ /* This should never be reached. Every opcode should end with DISPATCH() or goto error. */ Py_UNREACHABLE(); +#endif /* Py_TAIL_CALL_INTERP */ /* BEGIN LABELS */ pop_4_error: @@ -8644,7 +8647,14 @@ lltrace_resume_frame(frame); } #endif + // This is a little complicated... + // If we are in a tail call handler, we want to tail call (DISPATCH). + // If we're not then we need the shim frame. + #if defined(Py_TAIL_CALL_INTERP) && !defined(IN_TAIL_CALL_INTERP) + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0); + #else DISPATCH(); + #endif } exit_unwind: @@ -8657,7 +8667,7 @@ frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; - if (frame == &entry_frame) { + if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { /* Restore previous frame and exit */ tstate->current_frame = frame->previous; tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 6469b8507617e2..3336ea34522ccf 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -10,6 +10,7 @@ #define IN_TAIL_CALL_INTERP 1 static inline PyObject *_TAIL_CALL_shim(TAIL_CALL_PARAMS); static py_tail_call_funcptr INSTRUCTION_TABLE[256]; + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_4_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_3_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); @@ -45,116 +46,101 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS) { - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, + /* Double-check exception status. */ + #ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif + } + #else + assert(_PyErr_Occurred(tstate)); + #endif - /* Log traceback info. */ - assert(!frame->is_entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } + /* Log traceback info. */ + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); TAIL_CALL(exception_unwind); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS) { - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - TAIL_CALL(exit_unwind); - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - TAIL_CALL(exception_unwind); - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - TAIL_CALL(exception_unwind); - } - /* Resume normal execution */ -#ifdef LLTRACE - if (frame->lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - -#ifdef Py_TAIL_CALL_INTERP -# ifdef IN_TAIL_CALL_INTERP - DISPATCH(); -# else -# ifdef LLTRACE - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0, lltrace); -# else - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0); -# endif -# endif -#else - DISPATCH(); -#endif + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); } - - TAIL_CALL(exit_unwind); + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + TAIL_CALL(exit_unwind); + } + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + TAIL_CALL(exception_unwind); + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + TAIL_CALL(exception_unwind); + } + /* Resume normal execution */ + #ifdef LLTRACE + if (frame->lltrace >= 5) { + lltrace_resume_frame(frame); + } + #endif + // This is a little complicated... + // If we are in a tail call handler, we want to tail call (DISPATCH). + // If we're not then we need the shim frame. + #if defined(Py_TAIL_CALL_INTERP) && !defined(IN_TAIL_CALL_INTERP) + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0); + #else + DISPATCH(); + #endif } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS) { assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); - assert(!frame->is_entry_frame); + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; - if (frame->is_entry_frame) { + if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { /* Restore previous frame and exit */ tstate->current_frame = frame->previous; tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return NULL; } - TAIL_CALL(resume_with_error); } @@ -163,8 +149,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_resume_with_error(TAIL_CALL_PARA next_instr = frame->instr_ptr; stack_pointer = _PyFrame_GetStackPointer(frame); TAIL_CALL(error); - -/* END_BASE_INTERPRETER */ } diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index d76d66be5f467a..c3c386458c1fce 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -651,6 +651,19 @@ def emit_tokens( raise analysis_error(ex.args[0], rbrace) from None return storage + def emit_tokens_simple( + self, + tokens: list[Token] + ) -> Storage: + tkn_iter = TokenIterator(tokens) + self.out.start_line() + for tkn in tkn_iter: + if tkn.text in self._replacers: + self._replacers[tkn.text](tkn, tkn_iter, None, None, None) + continue + self.out.emit(tkn) + + def emit(self, txt: str | Token) -> None: self.out.emit(txt) diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index d5fbaca75e5cc6..27262d0ebd5f72 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -176,6 +176,8 @@ def generate_tier1( #endif #define TIER_ONE 1 +#ifndef Py_TAIL_CALL_INTERP + #if !USE_COMPUTED_GOTOS dispatch_opcode: switch (opcode) @@ -207,6 +209,7 @@ def generate_tier1( /* This should never be reached. Every opcode should end with DISPATCH() or goto error. */ Py_UNREACHABLE(); +#endif /* Py_TAIL_CALL_INTERP */ {LABEL_START_MARKER} """) generate_tier1_labels(analysis, outfile, lines) diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index f87c70212048ea..f752bd263ff465 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -34,12 +34,8 @@ DEFAULT_INPUT = ROOT / "Python/bytecodes.c" DEFAULT_OUTPUT = ROOT / "Python/generated_tail_call_handlers.c.h" -DEFAULT_CEVAL_INPUT = ROOT / "Python/ceval.c" - FOOTER = "#undef TIER_ONE\n#undef IN_TAIL_CALL_INTERP\n" -TARGET_LABEL = "TAIL_CALL_TARGET" - class TailCallEmitter(Emitter): def __init__(self, out: CWriter, analysis: Analysis): @@ -71,41 +67,77 @@ def go_to_instruction( self.emit(f"Py_MUSTTAIL return (INSTRUCTION_TABLE[{name.text}])(frame, stack_pointer, tstate, next_instr - 1 - {size}, opcode, oparg);\n") return True -def generate_label_handlers(infile: TextIO, outfile: TextIO) -> list[str]: - out = CWriter(outfile, 0, False) - str_in = infile.read() - # https://stackoverflow.com/questions/8303488/regex-to-match-any-character-including-new-lines - eval_framedefault = re.findall(r"_PyEval_EvalFrameDefault\(.*\)\n({[\s\S]*\/\* END_BASE_INTERPRETER \*\/)", str_in)[0] - function_protos = re.findall(rf"{TARGET_LABEL}\((\w+)\):", eval_framedefault) - for proto in function_protos: - out.emit(f"{function_proto(proto)};\n") - out.emit("\n") - lines = iter(eval_framedefault[eval_framedefault.find(TARGET_LABEL):].split("\n")) - next(lines) - for i in range(len(function_protos)): - curr_proto = function_protos[i] - fallthrough_proto = function_protos[i + 1] if i + 1 < len(function_protos) else None - out.emit(function_proto(curr_proto)) - out.emit("\n") - out.emit("{\n") - for line in lines: - if TARGET_LABEL in line: +class TailCallLabelsEmitter(Emitter): + def __init__(self, out: CWriter): + super().__init__(out) + self._replacers = { + 'goto': self.goto, + } + + def go_to_instruction( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + next(tkn_iter) + name = next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + assert name.kind == "IDENTIFIER" + self.emit("\n") + inst = self.analysis.instructions[name.text] + fam = None + # Search for the family (if any) + for family_name, family in self.analysis.families.items(): + if inst.name == family_name: + fam = family break - if label := re.findall(r"goto (\w+);", line): - out.emit(f"TAIL_CALL({label[0]});\n") - else: - out.emit_text(line) - out.emit("\n") - if fallthrough_proto: - out.emit(f"TAIL_CALL({fallthrough_proto});\n") - out.emit("}\n") - out.emit("\n") - return function_protos + size = fam.size if fam is not None else 0 + self.emit(f"Py_MUSTTAIL return (INSTRUCTION_TABLE[{name.text}])(frame, stack_pointer, tstate, next_instr - 1 - {size}, opcode, oparg);\n") + return True + + def goto( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + # Only labels need to replace their gotos with tail calls. + # We can't do this in normal code due to GCC's escape analysis + # complaining. + name = next(tkn_iter) + next(tkn_iter) + assert name.kind == "IDENTIFIER" + self.out.emit("\n") + self.emit(f"TAIL_CALL({name.text});\n") + def function_proto(name: str) -> str: return f"Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_{name}(TAIL_CALL_PARAMS)" +def generate_label_handlers( + analysis: Analysis, outfile: TextIO, lines: bool +) -> None: + out = CWriter(outfile, 0, lines) + emitter = TailCallLabelsEmitter(out) + emitter.emit("\n") + for name in analysis.labels: + emitter.emit(f"{function_proto(name)};\n") + emitter.emit("\n") + for name, label in analysis.labels.items(): + emitter.emit(f"{function_proto(name)}\n") + emitter.emit_tokens_simple(label.body) + + emitter.emit("\n") + emitter.emit("\n") + + def generate_tier1( filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool ) -> None: @@ -123,8 +155,7 @@ def generate_tier1( out.emit("static inline PyObject *_TAIL_CALL_shim(TAIL_CALL_PARAMS);\n") out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256];\n"); - with open(DEFAULT_CEVAL_INPUT, "r") as infile: - err_labels = generate_label_handlers(infile, outfile) + generate_label_handlers(analysis, outfile, lines) emitter = TailCallEmitter(out, analysis) out.emit("\n") @@ -146,7 +177,7 @@ def generate_tier1( # at the branch also produces the same. # Furthermore, this is required to make GCC 15's escape analysis happy # as written above. - for err_label in err_labels: + for err_label in analysis.labels.keys(): out.emit(f"{err_label}:\n") out.emit(f"TAIL_CALL({err_label});\n") out.start_line() From 09065eee26e15f359cb2c19d6dcf15dfaeb1a304 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 25 Jan 2025 15:48:45 +0800 Subject: [PATCH 073/110] refactor --- Python/ceval.c | 13 +++---------- Python/ceval_macros.h | 23 +---------------------- 2 files changed, 4 insertions(+), 32 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 86d69ffc39d37a..c0f9a05994c68a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -808,9 +808,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #ifdef Py_STATS int lastopcode = 0; #endif +#ifndef Py_TAIL_CALL_INTERP uint8_t opcode; /* Current opcode */ int oparg; /* Current opcode argument, if any */ - +#endif _PyInterpreterFrame entry_frame; @@ -890,15 +891,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int resume_frame: stack_pointer = _PyFrame_GetStackPointer(frame); -#ifdef LLTRACE - { - int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); - frame->lltrace = lltrace; - if (lltrace < 0) { - goto exit_unwind; - } - } -#endif + LLTRACE_RESUME_FRAME(); #ifdef Py_DEBUG /* _PyEval_EvalFrameDefault() must not be called with an exception set, diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 83718f64025884..dd0a2310f5de36 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -144,7 +144,6 @@ do { \ } #ifdef Py_TAIL_CALL_INTERP -#ifdef LLTRACE #define DISPATCH_INLINED(NEW_FRAME) \ do { \ assert(tstate->interp->eval_frame == NULL); \ @@ -157,31 +156,11 @@ do { \ } \ next_instr = frame->instr_ptr; \ stack_pointer = _PyFrame_GetStackPointer(frame); \ - lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); \ - if (lltrace < 0) { \ - goto exit_unwind; \ - } \ + LLTRACE_RESUME_FRAME(); \ NEXTOPARG(); \ DISPATCH_GOTO(); \ } while (0) #else -#define DISPATCH_INLINED(NEW_FRAME) \ -do { \ - assert(tstate->interp->eval_frame == NULL); \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - assert((NEW_FRAME)->previous == frame); \ - frame = tstate->current_frame = (NEW_FRAME); \ - CALL_STAT_INC(inlined_py_calls); \ - if (_Py_EnterRecursivePy(tstate)) { \ - goto exit_unwind; \ - } \ - next_instr = frame->instr_ptr; \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - NEXTOPARG(); \ - DISPATCH_GOTO(); \ - } while (0) -#endif -#else #define DISPATCH_INLINED(NEW_FRAME) \ do { \ assert(tstate->interp->eval_frame == NULL); \ From 08ce01a31548b5e277488d500687ce2bd8a7bac1 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 25 Jan 2025 15:55:38 +0800 Subject: [PATCH 074/110] Address review --- Python/ceval_macros.h | 4 +- Python/generated_cases.c.h | 452 ++++++++++----------- Python/generated_tail_call_handlers.c.h | 452 ++++++++++----------- Tools/cases_generator/generators_common.py | 2 +- Tools/jit/template.c | 8 - 5 files changed, 455 insertions(+), 463 deletions(-) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index dd0a2310f5de36..53fd30145c32ce 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -286,7 +286,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { #endif #ifdef Py_TAIL_CALL_INTERP -# define DEOPT_IF(COND, INSTNAME, SIZE) \ +# define GO_TO_INSTRUCTION_IF(COND, INSTNAME, SIZE) \ if ((COND)) { \ /* This is only a single jump on release builds! */ \ UPDATE_MISS_STATS((INSTNAME)); \ @@ -295,7 +295,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { return (INSTRUCTION_TABLE[INSTNAME])(frame, stack_pointer, tstate, next_instr - 1 - SIZE, opcode, oparg); \ } #else -# define DEOPT_IF(COND, INSTNAME, SIZE) \ +# define GO_TO_INSTRUCTION_IF(COND, INSTNAME, SIZE) \ if ((COND)) { \ /* This is only a single jump on release builds! */ \ UPDATE_MISS_STATS((INSTNAME)); \ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d443c615e33520..9d920fdfaa19ca 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -82,8 +82,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_ADD_FLOAT @@ -120,8 +120,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_ADD_INT @@ -157,8 +157,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_ADD_UNICODE @@ -202,7 +202,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int res = d->guard(left_o, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); - DEOPT_IF(!res, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!res, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip -4 cache entry */ // _BINARY_OP_EXTEND @@ -239,8 +239,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_INPLACE_ADD_UNICODE @@ -257,7 +257,7 @@ next_oparg = CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); - DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -303,8 +303,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_FLOAT @@ -341,8 +341,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_INT @@ -378,8 +378,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_FLOAT @@ -416,8 +416,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_INT @@ -549,7 +549,7 @@ dict_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -583,22 +583,22 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); } // _BINARY_SUBSCR_CHECK_FUNC { container = stack_pointer[-2]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(getitem_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); assert(PyFunction_Check(getitem_o)); uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); } @@ -643,19 +643,19 @@ list_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!PyList_CheckExact(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; #ifdef Py_GIL_DISABLED _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); stack_pointer = _PyFrame_GetStackPointer(frame); - DEOPT_IF(res_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(res_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); #else - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); @@ -683,14 +683,14 @@ str_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); @@ -715,12 +715,12 @@ tuple_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); @@ -1112,7 +1112,7 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_AND_ALLOCATE_OBJECT { @@ -1123,17 +1123,17 @@ self = &stack_pointer[-1 - oparg]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL, INLINE_CACHE_ENTRIES_CALL); assert(tp->tp_new == PyBaseObject_Type.tp_new); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); assert(tp->tp_alloc == PyType_GenericAlloc); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *self_o = PyType_GenericAlloc(tp, 0); @@ -1210,14 +1210,14 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS { null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { @@ -1236,9 +1236,9 @@ callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_EXACT_ARGS { @@ -1247,15 +1247,15 @@ assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_PY_EXACT_ARGS { @@ -1313,7 +1313,7 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_METHOD_VERSION { @@ -1321,11 +1321,11 @@ callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyFunction_Check(func), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(((PyFunctionObject *)func)->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); } // _EXPAND_METHOD { @@ -1414,7 +1414,7 @@ self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; int total_args = oparg; _PyStackRef *arguments = args; @@ -1422,7 +1422,7 @@ arguments--; total_args++; } - DEOPT_IF(tp->tp_vectorcall == NULL, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(tp->tp_vectorcall == NULL, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { @@ -1499,8 +1499,8 @@ arguments--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ @@ -1583,8 +1583,8 @@ arguments--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); /* res = func(self, arguments, nargs, kwnames) */ _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1667,11 +1667,11 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; @@ -1940,9 +1940,9 @@ arguments--; total_args++; } - DEOPT_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(callable_o != interp->callable_cache.isinstance, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyStackRef cls_stackref = arguments[1]; _PyStackRef inst_stackref = arguments[0]; @@ -2139,7 +2139,7 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CHECK_METHOD_VERSION_KW { @@ -2147,11 +2147,11 @@ callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + GO_TO_INSTRUCTION_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + GO_TO_INSTRUCTION_IF(!PyFunction_Check(func), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + GO_TO_INSTRUCTION_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null[0]), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _EXPAND_METHOD_KW { @@ -2245,8 +2245,8 @@ { callable = &stack_pointer[-3 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + GO_TO_INSTRUCTION_IF(PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + GO_TO_INSTRUCTION_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CALL_KW_NON_PY { @@ -2336,16 +2336,16 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CHECK_FUNCTION_VERSION_KW { callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + GO_TO_INSTRUCTION_IF(!PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + GO_TO_INSTRUCTION_IF(func->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _PY_FRAME_KW { @@ -2429,9 +2429,9 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.len, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(callable_o != interp->callable_cache.len, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyStackRef arg_stackref = args[0]; PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); @@ -2472,10 +2472,10 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(callable_o != interp->callable_cache.list_append, CALL, INLINE_CACHE_ENTRIES_CALL); assert(self_o != NULL); - DEOPT_IF(!PyList_Check(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(!LOCK_OBJECT(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyList_Check(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); @@ -2518,11 +2518,11 @@ } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(meth->ml_flags != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -2603,12 +2603,12 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *d_type = method->d_common.d_type; PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(self, d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -2688,16 +2688,16 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; _PyStackRef self_stackref = args[0]; PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(meth->ml_flags != METH_NOARGS, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -2760,15 +2760,15 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(meth->ml_flags != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); _PyStackRef arg_stackref = args[1]; _PyStackRef self_stackref = args[0]; - DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; @@ -2827,8 +2827,8 @@ { callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CALL_NON_PY_GENERAL { @@ -2912,16 +2912,16 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_EXACT_ARGS { @@ -2930,15 +2930,15 @@ assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_PY_EXACT_ARGS { @@ -2993,16 +2993,16 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _PY_FRAME_GENERAL { @@ -3078,8 +3078,8 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Str(arg_o); @@ -3129,8 +3129,8 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(callable_o != (PyObject *)&PyTuple_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PySequence_Tuple(arg_o); @@ -3178,8 +3178,8 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(callable_o != (PyObject *)&PyType_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); PyStackRef_CLOSE(arg); @@ -3381,8 +3381,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_FLOAT @@ -3419,16 +3419,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_INT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + GO_TO_INSTRUCTION_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + GO_TO_INSTRUCTION_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && _PyLong_DigitCount((PyLongObject *)right_o) <= 1); @@ -3461,8 +3461,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_STR @@ -3545,7 +3545,7 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); + GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(right_o), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); @@ -3573,7 +3573,7 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); + GO_TO_INSTRUCTION_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4080,14 +4080,14 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _FOR_ITER_GEN_FRAME { iter = stack_pointer[-1]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + GO_TO_INSTRUCTION_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + GO_TO_INSTRUCTION_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); STAT_INC(FOR_ITER, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, PyStackRef_None); @@ -4128,7 +4128,7 @@ // _ITER_CHECK_LIST { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + GO_TO_INSTRUCTION_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_LIST { @@ -4180,7 +4180,7 @@ { iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + GO_TO_INSTRUCTION_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_RANGE { @@ -4222,7 +4222,7 @@ // _ITER_CHECK_TUPLE { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + GO_TO_INSTRUCTION_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_TUPLE { @@ -5372,9 +5372,9 @@ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS @@ -5402,16 +5402,16 @@ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_TYPE_VERSION { uint32_t type_version = read_u32(&this_instr[4].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_CLASS { @@ -5437,17 +5437,17 @@ uint32_t func_version = read_u32(&this_instr[4].cache); PyObject *getattribute = read_obj(&this_instr[6].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(f->func_version != func_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( @@ -5474,14 +5474,14 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_MANAGED_OBJECT_HAS_VALUES { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { @@ -5489,10 +5489,10 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); - DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #ifdef Py_GIL_DISABLED if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { - DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } #else attr = PyStackRef_FromPyObjectNew(attr_o); @@ -5519,11 +5519,11 @@ owner = stack_pointer[-1]; uint32_t dict_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); mod_keys = keys; } // _LOAD_ATTR_MODULE_FROM_KEYS @@ -5534,11 +5534,11 @@ PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); // Clear mod_keys from stack in case we need to deopt - DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); if (!increfed) { - DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } #else Py_INCREF(attr_o); @@ -5566,7 +5566,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT @@ -5596,14 +5596,14 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyDictValues *ivs = _PyObject_InlineValues(owner_o); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_KEYS_VERSION { @@ -5611,7 +5611,7 @@ PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { @@ -5635,7 +5635,7 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_TYPE_VERSION { @@ -5643,7 +5643,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_PROPERTY_FRAME @@ -5652,10 +5652,10 @@ assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; PyCodeObject *code = (PyCodeObject *)f->func_code; - DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - DEOPT_IF(code->co_argcount != 1, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(code->co_kwonlyargcount, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(code->co_argcount != 1, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); new_frame->localsplus[0] = owner; @@ -5703,7 +5703,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_SLOT { @@ -5711,10 +5711,10 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **addr = (PyObject **)((char *)owner_o + index); PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); - DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); - DEOPT_IF(!increfed, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(!increfed, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #else attr = PyStackRef_FromPyObjectNew(attr_o); #endif @@ -5741,14 +5741,14 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_ATTR_WITH_HINT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(dict_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(PyDict_CheckExact((PyObject *)dict_o)); dict = dict_o; } @@ -5757,26 +5757,26 @@ uint16_t hint = read_u16(&this_instr[4].cache); PyObject *attr_o; if (!LOCK_OBJECT(dict)) { - DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } if (hint >= (size_t)dict->ma_keys->dk_nentries) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } attr_o = ep->me_value; if (attr_o == NULL) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); @@ -6131,18 +6131,18 @@ { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); assert(DK_IS_UNICODE(keys)); } // _GUARD_BUILTINS_VERSION_PUSH_KEYS { uint16_t version = read_u16(&this_instr[3].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); builtins_keys = keys; assert(DK_IS_UNICODE(builtins_keys)); } @@ -6151,10 +6151,10 @@ uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + GO_TO_INSTRUCTION_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + GO_TO_INSTRUCTION_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -6179,9 +6179,9 @@ { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); globals_keys = keys; assert(DK_IS_UNICODE(globals_keys)); } @@ -6191,10 +6191,10 @@ uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + GO_TO_INSTRUCTION_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + GO_TO_INSTRUCTION_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -6308,7 +6308,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_ATTR_METHOD_LAZY_DICT { @@ -6316,7 +6316,7 @@ char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(dict != NULL, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 1 cache entry */ // _LOAD_METHOD_LAZY_DICT @@ -6350,7 +6350,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_METHOD_NO_DICT @@ -6385,14 +6385,14 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyDictValues *ivs = _PyObject_InlineValues(owner_o); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_KEYS_VERSION { @@ -6400,7 +6400,7 @@ PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_METHOD_WITH_VALUES { @@ -6599,8 +6599,8 @@ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + GO_TO_INSTRUCTION_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + GO_TO_INSTRUCTION_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6736,8 +6736,8 @@ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_METHOD, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_METHOD, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + GO_TO_INSTRUCTION_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_METHOD, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + GO_TO_INSTRUCTION_IF(!PyType_Check(class), LOAD_SUPER_METHOD, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -7263,15 +7263,15 @@ INSTRUCTION_STATS(RESUME_CHECK); static_assert(0 == 0, "incorrect cache size"); #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME, 0); + GO_TO_INSTRUCTION_IF(_Py_emscripten_signal_clock == 0, RESUME, 0); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME, 0); + GO_TO_INSTRUCTION_IF(eval_breaker != version, RESUME, 0); #ifdef Py_GIL_DISABLED - DEOPT_IF(frame->tlbc_index != + GO_TO_INSTRUCTION_IF(frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME, 0); #endif DISPATCH(); @@ -7440,15 +7440,15 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, SEND, INLINE_CACHE_ENTRIES_SEND); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, SEND, INLINE_CACHE_ENTRIES_SEND); } // _SEND_GEN_FRAME { v = stack_pointer[-1]; receiver = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND, INLINE_CACHE_ENTRIES_SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND, INLINE_CACHE_ENTRIES_SEND); + GO_TO_INSTRUCTION_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND, INLINE_CACHE_ENTRIES_SEND); + GO_TO_INSTRUCTION_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND, INLINE_CACHE_ENTRIES_SEND); STAT_INC(SEND, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, v); @@ -7638,11 +7638,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); PyTypeObject *tp = Py_TYPE(owner_o); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } } // _GUARD_DORV_NO_DICT @@ -7653,7 +7653,7 @@ if (_PyObject_GetManagedDict(owner_o) || !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } } // _STORE_ATTR_INSTANCE_VALUE @@ -7696,14 +7696,14 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } // _STORE_ATTR_SLOT { value = stack_pointer[-2]; uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; @@ -7733,7 +7733,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } // _STORE_ATTR_WITH_HINT { @@ -7742,12 +7742,12 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); - DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(dict == NULL, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(dict), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); #ifdef Py_GIL_DISABLED if (dict != _PyObject_GetManagedDict(owner_o)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } #endif assert(PyDict_CheckExact((PyObject *)dict)); @@ -7755,17 +7755,17 @@ if (hint >= (size_t)dict->ma_keys->dk_nentries || !DK_IS_UNICODE(dict->ma_keys)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } PyObject *old_value = ep->me_value; if (old_value == NULL) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } _PyFrame_SetStackPointer(frame, stack_pointer); _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); @@ -8001,7 +8001,7 @@ dict_st = stack_pointer[-2]; value = stack_pointer[-3]; PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(dict), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, @@ -8029,16 +8029,16 @@ value = stack_pointer[-3]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + GO_TO_INSTRUCTION_IF(!PyList_CheckExact(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + GO_TO_INSTRUCTION_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); // Ensure index < len(list) if (index >= PyList_GET_SIZE(list)) { UNLOCK_OBJECT(list); - DEOPT_IF(true, STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + GO_TO_INSTRUCTION_IF(true, STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); } STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); @@ -8128,7 +8128,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); } // _REPLACE_WITH_TRUE { @@ -8149,7 +8149,7 @@ /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; - DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + GO_TO_INSTRUCTION_IF(!PyStackRef_BoolCheck(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); DISPATCH(); } @@ -8165,7 +8165,7 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value_o)) { assert(_Py_IsImmortal(value_o)); @@ -8190,7 +8190,7 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + GO_TO_INSTRUCTION_IF(!PyList_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; PyStackRef_CLOSE(value); @@ -8209,7 +8209,7 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + GO_TO_INSTRUCTION_IF(!PyStackRef_IsNone(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); res = PyStackRef_False; stack_pointer[-1] = res; @@ -8227,7 +8227,7 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); if (value_o == &_Py_STR(empty)) { assert(_Py_IsImmortal(value_o)); @@ -8363,11 +8363,11 @@ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); - DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + GO_TO_INSTRUCTION_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); if (PyList_GET_SIZE(seq_o) != oparg) { UNLOCK_OBJECT(seq_o); - DEOPT_IF(true, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + GO_TO_INSTRUCTION_IF(true, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq_o); @@ -8392,8 +8392,8 @@ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + GO_TO_INSTRUCTION_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + GO_TO_INSTRUCTION_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { @@ -8417,8 +8417,8 @@ seq = stack_pointer[-1]; assert(oparg == 2); PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + GO_TO_INSTRUCTION_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + GO_TO_INSTRUCTION_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 3336ea34522ccf..2b56dfba364eea 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -236,8 +236,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PA left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_ADD_FLOAT @@ -292,8 +292,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARA left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_ADD_INT @@ -347,8 +347,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_ADD_UNICODE @@ -410,7 +410,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAM _PyFrame_SetStackPointer(frame, stack_pointer); int res = d->guard(left_o, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); - DEOPT_IF(!res, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!res, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip -4 cache entry */ // _BINARY_OP_EXTEND @@ -465,8 +465,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TA left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_INPLACE_ADD_UNICODE @@ -483,7 +483,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TA next_oparg = CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); - DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -547,8 +547,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CA left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_FLOAT @@ -603,8 +603,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_INT @@ -658,8 +658,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CA left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_FLOAT @@ -714,8 +714,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); } /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_INT @@ -901,7 +901,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PAR dict_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -953,22 +953,22 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); } // _BINARY_SUBSCR_CHECK_FUNC { container = stack_pointer[-2]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(getitem_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); assert(PyFunction_Check(getitem_o)); uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); } @@ -1031,19 +1031,19 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL list_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!PyList_CheckExact(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; #ifdef Py_GIL_DISABLED _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); stack_pointer = _PyFrame_GetStackPointer(frame); - DEOPT_IF(res_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(res_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); #else - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); @@ -1089,14 +1089,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_ str_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); @@ -1139,12 +1139,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CAL tuple_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + GO_TO_INSTRUCTION_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); @@ -1698,7 +1698,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_C /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_AND_ALLOCATE_OBJECT { @@ -1709,17 +1709,17 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_C self = &stack_pointer[-1 - oparg]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL, INLINE_CACHE_ENTRIES_CALL); assert(tp->tp_new == PyBaseObject_Type.tp_new); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); assert(tp->tp_alloc == PyType_GenericAlloc); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *self_o = PyType_GenericAlloc(tp, 0); @@ -1814,14 +1814,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS { null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { @@ -1840,9 +1840,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_EXACT_ARGS { @@ -1851,15 +1851,15 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_PY_EXACT_ARGS { @@ -1935,7 +1935,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_C /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_METHOD_VERSION { @@ -1943,11 +1943,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_C callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyFunction_Check(func), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(((PyFunctionObject *)func)->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); } // _EXPAND_METHOD { @@ -2054,7 +2054,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; int total_args = oparg; _PyStackRef *arguments = args; @@ -2062,7 +2062,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR arguments--; total_args++; } - DEOPT_IF(tp->tp_vectorcall == NULL, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(tp->tp_vectorcall == NULL, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { @@ -2157,8 +2157,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARA arguments--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ @@ -2259,8 +2259,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS( arguments--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); /* res = func(self, arguments, nargs, kwnames) */ _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2361,11 +2361,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS) args--; total_args++; } - DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; @@ -2706,9 +2706,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS arguments--; total_args++; } - DEOPT_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(callable_o != interp->callable_cache.isinstance, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyStackRef cls_stackref = arguments[1]; _PyStackRef inst_stackref = arguments[0]; @@ -2941,7 +2941,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_P /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CHECK_METHOD_VERSION_KW { @@ -2949,11 +2949,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_P callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + GO_TO_INSTRUCTION_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + GO_TO_INSTRUCTION_IF(!PyFunction_Check(func), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + GO_TO_INSTRUCTION_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null[0]), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _EXPAND_METHOD_KW { @@ -3065,8 +3065,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) { callable = &stack_pointer[-3 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + GO_TO_INSTRUCTION_IF(PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + GO_TO_INSTRUCTION_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CALL_KW_NON_PY { @@ -3174,16 +3174,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _CHECK_FUNCTION_VERSION_KW { callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + GO_TO_INSTRUCTION_IF(!PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + GO_TO_INSTRUCTION_IF(func->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); } // _PY_FRAME_KW { @@ -3285,9 +3285,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.len, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(callable_o != interp->callable_cache.len, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyStackRef arg_stackref = args[0]; PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); @@ -3346,10 +3346,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAM PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(callable_o != interp->callable_cache.list_append, CALL, INLINE_CACHE_ENTRIES_CALL); assert(self_o != NULL); - DEOPT_IF(!PyList_Check(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(!LOCK_OBJECT(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyList_Check(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); @@ -3410,11 +3410,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(meth->ml_flags != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -3513,12 +3513,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); PyTypeObject *d_type = method->d_common.d_type; PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(self, d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -3616,16 +3616,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TA args--; total_args++; } - DEOPT_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; _PyStackRef self_stackref = args[0]; PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(meth->ml_flags != METH_NOARGS, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -3706,15 +3706,15 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CA total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(meth->ml_flags != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); _PyStackRef arg_stackref = args[1]; _PyStackRef self_stackref = args[0]; - DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; @@ -3791,8 +3791,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PA { callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CALL_NON_PY_GENERAL { @@ -3894,16 +3894,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PAR /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_EXACT_ARGS { @@ -3912,15 +3912,15 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PAR assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); } // _INIT_CALL_PY_EXACT_ARGS { @@ -3993,16 +3993,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); } // _PY_FRAME_GENERAL { @@ -4096,8 +4096,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS){ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Str(arg_o); @@ -4165,8 +4165,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(callable_o != (PyObject *)&PyTuple_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PySequence_Tuple(arg_o); @@ -4232,8 +4232,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS){ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); - DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); + GO_TO_INSTRUCTION_IF(callable_o != (PyObject *)&PyType_Type, CALL, INLINE_CACHE_ENTRIES_CALL); STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); PyStackRef_CLOSE(arg); @@ -4525,8 +4525,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAM left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_FLOAT @@ -4581,16 +4581,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS) left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_INT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + GO_TO_INSTRUCTION_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + GO_TO_INSTRUCTION_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && _PyLong_DigitCount((PyLongObject *)right_o) <= 1); @@ -4641,8 +4641,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS) left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_STR @@ -4761,7 +4761,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAM left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); + GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(right_o), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); @@ -4807,7 +4807,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); + GO_TO_INSTRUCTION_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5692,14 +5692,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS){ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _FOR_ITER_GEN_FRAME { iter = stack_pointer[-1]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + GO_TO_INSTRUCTION_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + GO_TO_INSTRUCTION_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); STAT_INC(FOR_ITER, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, PyStackRef_None); @@ -5758,7 +5758,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS){ // _ITER_CHECK_LIST { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + GO_TO_INSTRUCTION_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_LIST { @@ -5828,7 +5828,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS) { iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + GO_TO_INSTRUCTION_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_RANGE { @@ -5888,7 +5888,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS) // _ITER_CHECK_TUPLE { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + GO_TO_INSTRUCTION_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); } // _ITER_JUMP_TUPLE { @@ -7722,9 +7722,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS @@ -7770,16 +7770,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_C owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_TYPE_VERSION { uint32_t type_version = read_u32(&this_instr[4].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_CLASS { @@ -7823,17 +7823,17 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDE uint32_t func_version = read_u32(&this_instr[4].cache); PyObject *getattribute = read_obj(&this_instr[6].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(f->func_version != func_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( @@ -7878,14 +7878,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_MANAGED_OBJECT_HAS_VALUES { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { @@ -7893,10 +7893,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); - DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #ifdef Py_GIL_DISABLED if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { - DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } #else attr = PyStackRef_FromPyObjectNew(attr_o); @@ -7941,11 +7941,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM owner = stack_pointer[-1]; uint32_t dict_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); mod_keys = keys; } // _LOAD_ATTR_MODULE_FROM_KEYS @@ -7956,11 +7956,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); // Clear mod_keys from stack in case we need to deopt - DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); if (!increfed) { - DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } #else Py_INCREF(attr_o); @@ -8006,7 +8006,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT( uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT @@ -8054,14 +8054,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VAL uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyDictValues *ivs = _PyObject_InlineValues(owner_o); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_KEYS_VERSION { @@ -8069,7 +8069,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VAL PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { @@ -8111,7 +8111,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_TYPE_VERSION { @@ -8119,7 +8119,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_PROPERTY_FRAME @@ -8128,10 +8128,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; PyCodeObject *code = (PyCodeObject *)f->func_code; - DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - DEOPT_IF(code->co_argcount != 1, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(code->co_kwonlyargcount, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(code->co_argcount != 1, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); new_frame->localsplus[0] = owner; @@ -8197,7 +8197,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_ATTR_SLOT { @@ -8205,10 +8205,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **addr = (PyObject **)((char *)owner_o + index); PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); - DEOPT_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); - DEOPT_IF(!increfed, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(!increfed, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); #else attr = PyStackRef_FromPyObjectNew(attr_o); #endif @@ -8253,14 +8253,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_ATTR_WITH_HINT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(dict_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); assert(PyDict_CheckExact((PyObject *)dict_o)); dict = dict_o; } @@ -8269,26 +8269,26 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA uint16_t hint = read_u16(&this_instr[4].cache); PyObject *attr_o; if (!LOCK_OBJECT(dict)) { - DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } if (hint >= (size_t)dict->ma_keys->dk_nentries) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } attr_o = ep->me_value; if (attr_o == NULL) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); @@ -8895,18 +8895,18 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); assert(DK_IS_UNICODE(keys)); } // _GUARD_BUILTINS_VERSION_PUSH_KEYS { uint16_t version = read_u16(&this_instr[3].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); builtins_keys = keys; assert(DK_IS_UNICODE(builtins_keys)); } @@ -8915,10 +8915,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + GO_TO_INSTRUCTION_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + GO_TO_INSTRUCTION_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -8961,9 +8961,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PAR { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); globals_keys = keys; assert(DK_IS_UNICODE(globals_keys)); } @@ -8973,10 +8973,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PAR uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + GO_TO_INSTRUCTION_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + GO_TO_INSTRUCTION_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -9144,7 +9144,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_METHOD_LAZY_DICT(TAIL_CALL_ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _CHECK_ATTR_METHOD_LAZY_DICT { @@ -9152,7 +9152,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_METHOD_LAZY_DICT(TAIL_CALL_ char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(dict != NULL, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 1 cache entry */ // _LOAD_METHOD_LAZY_DICT @@ -9204,7 +9204,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_METHOD_NO_DICT(TAIL_CALL_PA uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_METHOD_NO_DICT @@ -9257,14 +9257,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_METHOD_WITH_VALUES(TAIL_CAL uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyDictValues *ivs = _PyObject_InlineValues(owner_o); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _GUARD_KEYS_VERSION { @@ -9272,7 +9272,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_METHOD_WITH_VALUES(TAIL_CAL PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); } // _LOAD_METHOD_WITH_VALUES { @@ -9561,8 +9561,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_P PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + GO_TO_INSTRUCTION_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + GO_TO_INSTRUCTION_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9734,8 +9734,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_METHOD_METHOD(TAIL_CA PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_METHOD, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_METHOD, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + GO_TO_INSTRUCTION_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_METHOD, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + GO_TO_INSTRUCTION_IF(!PyType_Check(class), LOAD_SUPER_METHOD, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -10675,15 +10675,15 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS){ INSTRUCTION_STATS(RESUME_CHECK); static_assert(0 == 0, "incorrect cache size"); #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME, 0); + GO_TO_INSTRUCTION_IF(_Py_emscripten_signal_clock == 0, RESUME, 0); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME, 0); + GO_TO_INSTRUCTION_IF(eval_breaker != version, RESUME, 0); #ifdef Py_GIL_DISABLED - DEOPT_IF(frame->tlbc_index != + GO_TO_INSTRUCTION_IF(frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME, 0); #endif } @@ -10924,15 +10924,15 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS){ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, SEND, INLINE_CACHE_ENTRIES_SEND); + GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, SEND, INLINE_CACHE_ENTRIES_SEND); } // _SEND_GEN_FRAME { v = stack_pointer[-1]; receiver = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND, INLINE_CACHE_ENTRIES_SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND, INLINE_CACHE_ENTRIES_SEND); + GO_TO_INSTRUCTION_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND, INLINE_CACHE_ENTRIES_SEND); + GO_TO_INSTRUCTION_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND, INLINE_CACHE_ENTRIES_SEND); STAT_INC(SEND, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, v); @@ -11230,11 +11230,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_C uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); PyTypeObject *tp = Py_TYPE(owner_o); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } } // _GUARD_DORV_NO_DICT @@ -11245,7 +11245,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_C if (_PyObject_GetManagedDict(owner_o) || !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } } // _STORE_ATTR_INSTANCE_VALUE @@ -11306,14 +11306,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } // _STORE_ATTR_SLOT { value = stack_pointer[-2]; uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; @@ -11361,7 +11361,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } // _STORE_ATTR_WITH_HINT { @@ -11370,12 +11370,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); - DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(dict == NULL, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(dict), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); #ifdef Py_GIL_DISABLED if (dict != _PyObject_GetManagedDict(owner_o)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } #endif assert(PyDict_CheckExact((PyObject *)dict)); @@ -11383,17 +11383,17 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P if (hint >= (size_t)dict->ma_keys->dk_nentries || !DK_IS_UNICODE(dict->ma_keys)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } PyObject *old_value = ep->me_value; if (old_value == NULL) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); } _PyFrame_SetStackPointer(frame, stack_pointer); _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); @@ -11791,7 +11791,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARA dict_st = stack_pointer[-2]; value = stack_pointer[-3]; PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(dict), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, @@ -11837,16 +11837,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_ value = stack_pointer[-3]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + GO_TO_INSTRUCTION_IF(!PyList_CheckExact(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + GO_TO_INSTRUCTION_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); // Ensure index < len(list) if (index >= PyList_GET_SIZE(list)) { UNLOCK_OBJECT(list); - DEOPT_IF(true, STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + GO_TO_INSTRUCTION_IF(true, STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); } STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); @@ -11990,7 +11990,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PA uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); } // _REPLACE_WITH_TRUE { @@ -12029,7 +12029,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS){ /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; - DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + GO_TO_INSTRUCTION_IF(!PyStackRef_BoolCheck(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); } DISPATCH(); @@ -12063,7 +12063,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS){ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value_o)) { assert(_Py_IsImmortal(value_o)); @@ -12106,7 +12106,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS){ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + GO_TO_INSTRUCTION_IF(!PyList_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; PyStackRef_CLOSE(value); @@ -12143,7 +12143,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS){ /* Skip 2 cache entries */ value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + GO_TO_INSTRUCTION_IF(!PyStackRef_IsNone(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); res = PyStackRef_False; stack_pointer[-1] = res; @@ -12179,7 +12179,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS){ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); STAT_INC(TO_BOOL, hit); if (value_o == &_Py_STR(empty)) { assert(_Py_IsImmortal(value_o)); @@ -12423,11 +12423,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_P seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); - DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + GO_TO_INSTRUCTION_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); if (PyList_GET_SIZE(seq_o) != oparg) { UNLOCK_OBJECT(seq_o); - DEOPT_IF(true, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + GO_TO_INSTRUCTION_IF(true, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq_o); @@ -12470,8 +12470,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + GO_TO_INSTRUCTION_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + GO_TO_INSTRUCTION_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { @@ -12513,8 +12513,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_C seq = stack_pointer[-1]; assert(oparg == 2); PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + GO_TO_INSTRUCTION_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + GO_TO_INSTRUCTION_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index c3c386458c1fce..a74aff6b723f90 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -150,7 +150,7 @@ def deopt_if( storage: Storage, inst: Instruction | None, ) -> bool: - self.out.emit_at("DEOPT_IF", tkn) + self.out.emit_at("GO_TO_INSTRUCTION_IF", tkn) lparen = next(tkn_iter) self.emit(lparen) assert lparen.kind == "LPAREN" diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 95c90bda70f352..929f37d2dddea1 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -32,14 +32,6 @@ #undef CURRENT_OPERAND1 #define CURRENT_OPERAND1() (_operand1) -#undef DEOPT_IF -#define DEOPT_IF(COND, INSTNAME) \ - do { \ - if ((COND)) { \ - goto deoptimize; \ - } \ - } while (0) - #undef ENABLE_SPECIALIZATION #define ENABLE_SPECIALIZATION (0) From e7b791bd0a10640f21a2bf0628af4b9670fcebd8 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 25 Jan 2025 15:56:41 +0800 Subject: [PATCH 075/110] fix test --- Lib/test/test_generated_cases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index f53fd302bca3a0..a393ba37dfdde6 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -427,7 +427,7 @@ def test_predictions(self): INSTRUCTION_STATS(OP3); static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size"); _PyStackRef res; - DEOPT_IF(xxx, OP1, INLINE_CACHE_ENTRIES_OP1); + GO_TO_INSTRUCTION_IF(xxx, OP1, INLINE_CACHE_ENTRIES_OP1); res = Py_None; stack_pointer[-1] = res; DISPATCH(); From 7ea7ddc65ce424c94190526390b5430505ae6012 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 25 Jan 2025 15:59:20 +0800 Subject: [PATCH 076/110] Update generated_tail_call_handlers.c.h --- Python/generated_tail_call_handlers.c.h | 819 ++++++++++-------------- 1 file changed, 335 insertions(+), 484 deletions(-) diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 2b56dfba364eea..71dfc7b5a8fe88 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -1183,10 +1183,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS){ _PyStackRef *values; _PyStackRef list; values = &stack_pointer[-oparg]; - PyObject *list_o = _PyList_FromStackRefSteal(values, oparg); + PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); if (list_o == NULL) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); goto error; } list = PyStackRef_FromPyObjectSteal(list_o); @@ -1340,30 +1338,28 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SLICE); - _PyStackRef *args; + _PyStackRef start; + _PyStackRef stop; + _PyStackRef step = PyStackRef_NULL; _PyStackRef slice; - args = &stack_pointer[-oparg]; - assert(oparg == 2 || oparg == 3); - _PyStackRef start = args[0]; - _PyStackRef stop = args[1]; + if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } + stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; + start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); - PyObject * step_o = NULL; - if (oparg == 3) { - step_o = PyStackRef_AsPyObjectBorrow(args[2]); - } + PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } + PyStackRef_CLOSE(start); + PyStackRef_CLOSE(stop); + PyStackRef_XCLOSE(step); if (slice_o == NULL) { - stack_pointer += -oparg; + stack_pointer += -2 - ((oparg == 3) ? 1 : 0); assert(WITHIN_STACK_BOUNDS()); goto error; } slice = PyStackRef_FromPyObjectSteal(slice_o); - stack_pointer[-oparg] = slice; - stack_pointer += 1 - oparg; + stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; + stack_pointer += -1 - ((oparg == 3) ? 1 : 0); assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -1446,10 +1442,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS){ _PyStackRef *values; _PyStackRef tup; values = &stack_pointer[-oparg]; - PyObject *tup_o = _PyTuple_FromStackRefSteal(values, oparg); + PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); if (tup_o == NULL) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); goto error; } tup = PyStackRef_FromPyObjectSteal(tup_o); @@ -2433,18 +2427,18 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM (void)this_instr; _PyStackRef func; _PyStackRef callargs; - _PyStackRef kwargs_in; + _PyStackRef kwargs_in = PyStackRef_NULL; _PyStackRef tuple; - _PyStackRef kwargs_out; + _PyStackRef kwargs_out = PyStackRef_NULL; _PyStackRef func_st; _PyStackRef callargs_st; - _PyStackRef kwargs_st; + _PyStackRef kwargs_st = PyStackRef_NULL; _PyStackRef result; // _MAKE_CALLARGS_A_TUPLE { - kwargs_in = stack_pointer[-1]; - callargs = stack_pointer[-2]; - func = stack_pointer[-4]; + if (oparg & 1) { kwargs_in = stack_pointer[-(oparg & 1)]; } + callargs = stack_pointer[-1 - (oparg & 1)]; + func = stack_pointer[-3 - (oparg & 1)]; PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (PyTuple_CheckExact(callargs_o)) { tuple = callargs; @@ -2486,8 +2480,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM assert(PyTuple_CheckExact(callargs)); PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - stack_pointer[-2] = callargs_st; - stack_pointer[-1] = kwargs_st; + stack_pointer[-1 - (oparg & 1)] = callargs_st; + if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, @@ -2530,7 +2524,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -3; + stack_pointer += -2 - (oparg & 1); assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( @@ -2551,8 +2545,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM assert(PyTuple_CheckExact(callargs)); PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[-2] = callargs_st; - stack_pointer[-1] = kwargs_st; + stack_pointer[-1 - (oparg & 1)] = callargs_st; + if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); result_o = PyObject_Call(func, callargs, kwargs); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2562,7 +2556,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(callargs_st); PyStackRef_CLOSE(func_st); - if (result_o == NULL) goto pop_4_error; + if (result_o == NULL) { + stack_pointer += -3 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + goto error; + } result = PyStackRef_FromPyObjectSteal(result_o); } // _CHECK_PERIODIC @@ -2570,19 +2568,19 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-4] = result; - stack_pointer += -3; + stack_pointer[-3 - (oparg & 1)] = result; + stack_pointer += -2 - (oparg & 1); assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) goto error; - stack_pointer += 3; + stack_pointer += 2 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-4] = result; - stack_pointer += -3; + stack_pointer[-3 - (oparg & 1)] = result; + stack_pointer += -2 - (oparg & 1); assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -6886,36 +6884,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAI TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_METHOD(TAIL_CALL_PARAMS){ - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_METHOD); - /* Skip 1 cache entry */ - // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we - // don't want to specialize instrumented instructions - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - Py_MUSTTAIL return (INSTRUCTION_TABLE[LOAD_SUPER_METHOD])(frame, stack_pointer, tstate, next_instr - 1 - INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR, opcode, oparg); - } - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); -} - Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const prev_instr = frame->instr_ptr; @@ -7658,6 +7626,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ (void)this_instr; _PyStackRef owner; _PyStackRef attr; + _PyStackRef self_or_null = PyStackRef_NULL; // _SPECIALIZE_LOAD_ATTR { owner = stack_pointer[-1]; @@ -7665,7 +7634,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ (void)counter; #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadAttr(owner, next_instr, name); @@ -7679,15 +7648,50 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ /* Skip 8 cache entries */ // _LOAD_ATTR { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(owner); - if (attr_o == NULL) goto pop_1_error; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + PyObject *attr_o; + if (oparg & 1) { + /* Designed to work in tandem with CALL, pushes two values. */ + attr_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (is_meth) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr_o != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + } + else { + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + meth | NULL | arg1 | ... | argN + */ + PyStackRef_CLOSE(owner); + if (attr_o == NULL) goto pop_1_error; + self_or_null = PyStackRef_NULL; + } + } + else { + /* Classic, pushes one value. */ + _PyFrame_SetStackPointer(frame, stack_pointer); + attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(owner); + if (attr_o == NULL) goto pop_1_error; + /* We need to define self_or_null on all paths */ + self_or_null = PyStackRef_NULL; + } attr = PyStackRef_FromPyObjectSteal(attr_o); } stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); pop_4_error: @@ -7716,6 +7720,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _CHECK_ATTR_CLASS { @@ -7733,9 +7738,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); pop_4_error: @@ -7764,6 +7773,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_C static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _CHECK_ATTR_CLASS { @@ -7787,9 +7797,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_C STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); pop_4_error: @@ -7823,6 +7837,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDE uint32_t func_version = read_u32(&this_instr[4].cache); PyObject *getattribute = read_obj(&this_instr[6].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert((oparg & 1) == 0); GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); @@ -7835,7 +7850,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDE assert(code->co_argcount == 2); GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( tstate, PyStackRef_FromPyObjectNew(f), 2, frame); // Manipulate stack directly because we exit with DISPATCH_INLINED(). @@ -7871,6 +7886,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { @@ -7902,10 +7918,197 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); + null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS){ + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + } + // _CHECK_ATTR_METHOD_LAZY_DICT + { + uint16_t dictoffset = read_u16(&this_instr[4].cache); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); + /* This object has a __dict__, just not yet created */ + GO_TO_INSTRUCTION_IF(dict != NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + } + /* Skip 1 cache entry */ + // _LOAD_ATTR_METHOD_LAZY_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS){ + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS){ + { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + GO_TO_INSTRUCTION_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + } + // _LOAD_ATTR_METHOD_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + /* Cached method object */ + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); pop_4_error: @@ -7935,6 +8138,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM _PyStackRef owner; PyDictKeysObject *mod_keys; _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _CHECK_ATTR_MODULE_PUSH_KEYS { @@ -7967,10 +8171,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM attr = PyStackRef_FromPyObjectSteal(attr_o); #endif STAT_INC(LOAD_ATTR, hit); + null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); pop_4_error: @@ -8012,6 +8220,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT( // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT { PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -8074,6 +8283,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VAL // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); PyStackRef_CLOSE(owner); @@ -8125,6 +8335,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR // _LOAD_ATTR_PROPERTY_FRAME { PyObject *fget = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; PyCodeObject *code = (PyCodeObject *)f->func_code; @@ -8190,6 +8401,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { @@ -8213,10 +8425,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); + null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); pop_4_error: @@ -8246,6 +8462,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA _PyStackRef owner; PyDictObject *dict; _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { @@ -8275,7 +8492,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA UNLOCK_OBJECT(dict); GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); @@ -8293,10 +8510,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); UNLOCK_OBJECT(dict); + null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); pop_4_error: @@ -8831,13 +9052,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS){ _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; _PyStackRef *res; + _PyStackRef null = PyStackRef_NULL; // _SPECIALIZE_LOAD_GLOBAL { uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); @@ -8854,13 +9076,15 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS){ // _LOAD_GLOBAL { res = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*res)) goto error; + null = PyStackRef_NULL; } - stack_pointer += 1; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -8890,6 +9114,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyDictKeysObject *builtins_keys; _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_GLOBALS_VERSION { @@ -8924,9 +9149,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA res = PyStackRef_FromPyObjectSteal(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); + null = PyStackRef_NULL; } stack_pointer[0] = res; - stack_pointer += 1; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -8956,6 +9183,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PAR static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyDictKeysObject *globals_keys; _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_GLOBALS_VERSION_PUSH_KEYS { @@ -8982,9 +9210,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PAR res = PyStackRef_FromPyObjectSteal(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); + null = PyStackRef_NULL; } stack_pointer[0] = res; - stack_pointer += 1; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -9044,271 +9274,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS){ TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_METHOD(TAIL_CALL_PARAMS){ - { - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_METHOD); - PREDICTED_LOAD_METHOD:; - _Py_CODEUNIT* const this_instr = next_instr - 10; - (void)this_instr; - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self_or_null; - // _SPECIALIZE_LOAD_METHOD - { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadMethod(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_METHOD); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 8 cache entries */ - // _LOAD_METHOD - { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *attr_o; - /* Designed to work in tandem with CALL, pushes two values. */ - attr_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (is_meth) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - meth | self | arg1 | ... | argN - */ - assert(attr_o != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership - } - else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - meth | NULL | arg1 | ... | argN - */ - PyStackRef_CLOSE(owner); - if (attr_o == NULL) goto pop_1_error; - self_or_null = PyStackRef_NULL; - } - attr = PyStackRef_FromPyObjectSteal(attr_o); - } - stack_pointer[-1] = attr; - stack_pointer[0] = self_or_null; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_METHOD_LAZY_DICT(TAIL_CALL_PARAMS){ - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_METHOD_LAZY_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); - } - // _CHECK_ATTR_METHOD_LAZY_DICT - { - uint16_t dictoffset = read_u16(&this_instr[4].cache); - char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); - /* This object has a __dict__, just not yet created */ - GO_TO_INSTRUCTION_IF(dict != NULL, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); - } - /* Skip 1 cache entry */ - // _LOAD_METHOD_LAZY_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_METHOD_NO_DICT(TAIL_CALL_PARAMS){ - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_METHOD_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_METHOD_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_METHOD_WITH_VALUES(TAIL_CALL_PARAMS){ - { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_METHOD_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyDictValues *ivs = _PyObject_InlineValues(owner_o); - GO_TO_INSTRUCTION_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR); - } - // _LOAD_METHOD_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - /* Cached method object */ - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS){ { frame->instr_ptr = next_instr; next_instr += 1; @@ -9442,6 +9408,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS _PyStackRef class_st; _PyStackRef self_st; _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; // _SPECIALIZE_LOAD_SUPER_ATTR { class_st = stack_pointer[-2]; @@ -9449,10 +9416,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION_FT + int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 0); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } @@ -9466,7 +9434,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode >= MIN_INSTRUMENTED_OPCODE) { + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( @@ -9486,7 +9454,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode >= MIN_INSTRUMENTED_OPCODE) { + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9519,9 +9487,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) goto error; attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; } stack_pointer[0] = attr; - stack_pointer += 1; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -9596,130 +9566,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_P TAIL_CALL(resume_with_error); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_METHOD(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS){ { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_METHOD); - PREDICTED_LOAD_SUPER_METHOD:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef null; - // _SPECIALIZE_LOAD_SUPER_METHOD - { - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 1); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_SUPER_METHOD); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _LOAD_SUPER_ATTR - { - self_st = stack_pointer[-1]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode >= MIN_INSTRUMENTED_OPCODE) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - goto pop_3_error; - } - } - // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = {class, self}; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode >= MIN_INSTRUMENTED_OPCODE) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(super); - } - } - } - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - if (super == NULL) goto pop_3_error; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(super, name); - Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) goto error; - attr = PyStackRef_FromPyObjectSteal(attr_o); - } - // _PUSH_NULL - { - null = PyStackRef_NULL; - } - stack_pointer[0] = attr; - stack_pointer[1] = null; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_METHOD_METHOD(TAIL_CALL_PARAMS){ - { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_METHOD_METHOD); + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); _PyStackRef global_super_st; _PyStackRef class_st; @@ -9734,8 +9585,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_METHOD_METHOD(TAIL_CA PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(oparg & 1); - GO_TO_INSTRUCTION_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_METHOD, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); - GO_TO_INSTRUCTION_IF(!PyType_Check(class), LOAD_SUPER_METHOD, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + GO_TO_INSTRUCTION_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + GO_TO_INSTRUCTION_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -10424,9 +10275,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_NULL); - _PyStackRef null; - null = PyStackRef_NULL; - stack_pointer[0] = null; + _PyStackRef res; + res = PyStackRef_NULL; + stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } @@ -12794,7 +12645,6 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD, [INSTRUMENTED_LINE] = _TAIL_CALL_INSTRUMENTED_LINE, [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR, - [INSTRUMENTED_LOAD_SUPER_METHOD] = _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_METHOD, [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_INSTRUMENTED_NOT_TAKEN, [INSTRUMENTED_POP_ITER] = _TAIL_CALL_INSTRUMENTED_POP_ITER, [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE, @@ -12816,6 +12666,9 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE, + [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT, + [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT, + [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES, [LOAD_ATTR_MODULE] = _TAIL_CALL_LOAD_ATTR_MODULE, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, @@ -12838,17 +12691,12 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_LOAD_GLOBAL_BUILTIN, [LOAD_GLOBAL_MODULE] = _TAIL_CALL_LOAD_GLOBAL_MODULE, [LOAD_LOCALS] = _TAIL_CALL_LOAD_LOCALS, - [LOAD_METHOD] = _TAIL_CALL_LOAD_METHOD, - [LOAD_METHOD_LAZY_DICT] = _TAIL_CALL_LOAD_METHOD_LAZY_DICT, - [LOAD_METHOD_NO_DICT] = _TAIL_CALL_LOAD_METHOD_NO_DICT, - [LOAD_METHOD_WITH_VALUES] = _TAIL_CALL_LOAD_METHOD_WITH_VALUES, [LOAD_NAME] = _TAIL_CALL_LOAD_NAME, [LOAD_SMALL_INT] = _TAIL_CALL_LOAD_SMALL_INT, [LOAD_SPECIAL] = _TAIL_CALL_LOAD_SPECIAL, [LOAD_SUPER_ATTR] = _TAIL_CALL_LOAD_SUPER_ATTR, [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_LOAD_SUPER_ATTR_ATTR, - [LOAD_SUPER_METHOD] = _TAIL_CALL_LOAD_SUPER_METHOD, - [LOAD_SUPER_METHOD_METHOD] = _TAIL_CALL_LOAD_SUPER_METHOD_METHOD, + [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_LOAD_SUPER_ATTR_METHOD, [MAKE_CELL] = _TAIL_CALL_MAKE_CELL, [MAKE_FUNCTION] = _TAIL_CALL_MAKE_FUNCTION, [MAP_ADD] = _TAIL_CALL_MAP_ADD, @@ -12912,6 +12760,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, + [118] = _TAIL_CALL_UNKNOWN_OPCODE, + [119] = _TAIL_CALL_UNKNOWN_OPCODE, [120] = _TAIL_CALL_UNKNOWN_OPCODE, [121] = _TAIL_CALL_UNKNOWN_OPCODE, [122] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -12945,6 +12795,7 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, [233] = _TAIL_CALL_UNKNOWN_OPCODE, + [234] = _TAIL_CALL_UNKNOWN_OPCODE, }; #undef TIER_ONE #undef IN_TAIL_CALL_INTERP From 627fb59d832c5af42549e1407aecfd6f41baa7bf Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 25 Jan 2025 20:40:09 +0800 Subject: [PATCH 077/110] Address review by Hugo Co-Authored-By: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/tail-call.yml | 17 +++++++++-------- Lib/test/test_generated_cases.py | 1 - 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index 5310e49af7f3cf..c05bb1201ff95c 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -21,6 +21,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +env: + FORCE_COLOR: 1 + jobs: tail-call: name: ${{ matrix.target }} @@ -61,8 +64,7 @@ jobs: runner: ubuntu-24.04 - target: aarch64-unknown-linux-gnu/gcc architecture: aarch64 - # Forks don't have access to our paid AArch64 runners. These jobs are skipped below: - runner: ${{ github.repository_owner == 'python' && 'ubuntu-24.04-aarch64' || 'ubuntu-24.04' }} + runner: ubuntu-24.04-arm steps: - uses: actions/checkout@v4 with: @@ -71,7 +73,7 @@ jobs: with: python-version: '3.11' - - name: Native Windows (Debug) + - name: Native Windows (debug) if: runner.os == 'Windows' && matrix.architecture != 'ARM64' run: | choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 @@ -79,7 +81,7 @@ jobs: ./PCbuild/rt.bat -d -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 # No tests (yet): - - name: Emulated Windows (Release) + - name: Emulated Windows (release) if: runner.os == 'Windows' && matrix.architecture == 'ARM64' run: | choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 @@ -89,7 +91,7 @@ jobs: # This is a bug in the macOS runner image where the pre-installed Python is installed in the same # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes # the symlink to the pre-installed Python so that the Homebrew Python is used instead. - - name: Native macOS (Debug) + - name: Native macOS (debug) if: runner.os == 'macOS' run: | brew update @@ -102,9 +104,8 @@ jobs: make all --jobs 4 ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - name: Native Linux (Release) - # Forks don't have access to our paid AArch64 runners. Skip those: - if: runner.os == 'Linux' && (matrix.architecture == 'x86_64' || github.repository_owner == 'python') + - name: Native Linux (release) + if: runner.os == 'Linux' run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index b5d98d80b80c03..6873f766cc1e1e 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -4,7 +4,6 @@ import sys import tempfile import unittest -import textwrap from io import StringIO from test import support From 368f60ed4cae4280a7a8e70c1e39308f7f658987 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 25 Jan 2025 20:46:49 +0800 Subject: [PATCH 078/110] Make mypy happy --- Tools/cases_generator/generators_common.py | 4 +-- .../tier1_tail_call_generator.py | 26 +------------------ 2 files changed, 3 insertions(+), 27 deletions(-) diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index e1934e9f0a6905..9907c80de95d39 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -657,12 +657,12 @@ def emit_tokens( def emit_tokens_simple( self, tokens: list[Token] - ) -> Storage: + ) -> None: tkn_iter = TokenIterator(tokens) self.out.start_line() for tkn in tkn_iter: if tkn.text in self._replacers: - self._replacers[tkn.text](tkn, tkn_iter, None, None, None) + self._replacers[tkn.text](tkn, tkn_iter, None, None, None) # type: ignore[arg-type] continue self.out.emit(tkn) diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index f752bd263ff465..2c65b2a062fa0b 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -74,31 +74,6 @@ def __init__(self, out: CWriter): 'goto': self.goto, } - def go_to_instruction( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: Uop, - storage: Storage, - inst: Instruction | None, - ) -> bool: - next(tkn_iter) - name = next(tkn_iter) - next(tkn_iter) - next(tkn_iter) - assert name.kind == "IDENTIFIER" - self.emit("\n") - inst = self.analysis.instructions[name.text] - fam = None - # Search for the family (if any) - for family_name, family in self.analysis.families.items(): - if inst.name == family_name: - fam = family - break - size = fam.size if fam is not None else 0 - self.emit(f"Py_MUSTTAIL return (INSTRUCTION_TABLE[{name.text}])(frame, stack_pointer, tstate, next_instr - 1 - {size}, opcode, oparg);\n") - return True - def goto( self, tkn: Token, @@ -115,6 +90,7 @@ def goto( assert name.kind == "IDENTIFIER" self.out.emit("\n") self.emit(f"TAIL_CALL({name.text});\n") + return True From b7f5fafc156d9d791717e02a1c27b200fb451ea1 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 26 Jan 2025 19:43:29 +0800 Subject: [PATCH 079/110] Update generated_tail_call_handlers.c.h --- Python/generated_tail_call_handlers.c.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 71dfc7b5a8fe88..429847b19d2703 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -4282,7 +4282,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS) PyObject *match_o = NULL; PyObject *rest_o = NULL; _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, + int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, &match_o, &rest_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(exc_value_st); From 9f90bfeb21dc3106032bb42218ac97204b5f6fd5 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 27 Jan 2025 18:36:48 +0800 Subject: [PATCH 080/110] Update 2025-01-10-18-56-20.gh-issue-128563.baDvls.rst --- .../2025-01-10-18-56-20.gh-issue-128563.baDvls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst index 10110fd815f395..63d177053cae71 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst @@ -1 +1 @@ -A new type of interpreter has been added to CPython. This interpreter uses tail calls for its instruction handlers. Preliminary benchmark results suggest 7-11% geometric mean faster on pyperformance (depending on platform), and up to 30% faster on Python-intensive workloads. This interpreter currently only works on newer compilers, such as ``clang-19``. Other compilers will continue using the old interpreter. Patch by Ken Jin, with ideas by Garret Gu, Haoran Xu, and Josh Haberman. +A new type of interpreter has been added to CPython. This interpreter uses tail calls for its instruction handlers. Preliminary benchmark results suggest 7-11% geometric mean faster on pyperformance (depending on platform), and up to 30% faster on Python-intensive workloads. This interpreter currently only works on newer compilers, such as ``clang-19``. Other compilers will continue using the old interpreter. Patch by Ken Jin, with ideas by Mark Shannon, Garret Gu, Haoran Xu, and Josh Haberman. From db5114b2ff63965971381f5687b3db9b7fda7f10 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 27 Jan 2025 18:39:38 +0800 Subject: [PATCH 081/110] Cleanup --- Python/ceval_macros.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 53fd30145c32ce..df60caef49444c 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -96,9 +96,6 @@ # define DISPATCH_GOTO() goto dispatch_opcode #endif -#define TAIL_CALL_TARGET(name) name - - /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ #ifdef LLTRACE #define PRE_DISPATCH_GOTO() if (frame->lltrace >= 5) { \ From 5b79a66be425f7caafa47433321521b891a9af64 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 29 Jan 2025 22:48:13 +0800 Subject: [PATCH 082/110] Update generated_tail_call_handlers.c.h --- Python/generated_tail_call_handlers.c.h | 301 +++++++++++++++++++----- 1 file changed, 237 insertions(+), 64 deletions(-) diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 429847b19d2703..7c06ec7a6f37d0 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -390,6 +390,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_EXTEND); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -1338,28 +1339,24 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS){ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SLICE); - _PyStackRef start; - _PyStackRef stop; - _PyStackRef step = PyStackRef_NULL; + _PyStackRef *args; _PyStackRef slice; - if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } - stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; - start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; - PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); - PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); - PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); + args = &stack_pointer[-oparg]; + PyObject *start_o = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); + PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); - PyStackRef_CLOSE(start); - PyStackRef_CLOSE(stop); - PyStackRef_XCLOSE(step); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (slice_o == NULL) { - stack_pointer += -2 - ((oparg == 3) ? 1 : 0); + stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); goto error; } slice = PyStackRef_FromPyObjectSteal(slice_o); - stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; - stack_pointer += -1 - ((oparg == 3) ? 1 : 0); + stack_pointer[-oparg] = slice; + stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -1679,6 +1676,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -1795,6 +1793,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_C Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -1916,6 +1915,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -2427,18 +2427,18 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM (void)this_instr; _PyStackRef func; _PyStackRef callargs; - _PyStackRef kwargs_in = PyStackRef_NULL; + _PyStackRef kwargs_in; _PyStackRef tuple; - _PyStackRef kwargs_out = PyStackRef_NULL; + _PyStackRef kwargs_out; _PyStackRef func_st; _PyStackRef callargs_st; - _PyStackRef kwargs_st = PyStackRef_NULL; + _PyStackRef kwargs_st; _PyStackRef result; // _MAKE_CALLARGS_A_TUPLE { - if (oparg & 1) { kwargs_in = stack_pointer[-(oparg & 1)]; } - callargs = stack_pointer[-1 - (oparg & 1)]; - func = stack_pointer[-3 - (oparg & 1)]; + kwargs_in = stack_pointer[-1]; + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (PyTuple_CheckExact(callargs_o)) { tuple = callargs; @@ -2480,8 +2480,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM assert(PyTuple_CheckExact(callargs)); PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - stack_pointer[-1 - (oparg & 1)] = callargs_st; - if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; + stack_pointer[-2] = callargs_st; + stack_pointer[-1] = kwargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, @@ -2524,7 +2524,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -2 - (oparg & 1); + stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( @@ -2545,8 +2545,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM assert(PyTuple_CheckExact(callargs)); PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[-1 - (oparg & 1)] = callargs_st; - if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; + stack_pointer[-2] = callargs_st; + stack_pointer[-1] = kwargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); result_o = PyObject_Call(func, callargs, kwargs); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2556,11 +2556,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(callargs_st); PyStackRef_CLOSE(func_st); - if (result_o == NULL) { - stack_pointer += -3 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + if (result_o == NULL) goto pop_4_error; result = PyStackRef_FromPyObjectSteal(result_o); } // _CHECK_PERIODIC @@ -2568,19 +2564,19 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-3 - (oparg & 1)] = result; - stack_pointer += -2 - (oparg & 1); + stack_pointer[-4] = result; + stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) goto error; - stack_pointer += 2 + (oparg & 1); + stack_pointer += 3; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-3 - (oparg & 1)] = result; - stack_pointer += -2 - (oparg & 1); + stack_pointer[-4] = result; + stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -2925,6 +2921,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); @@ -3161,6 +3158,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_PY); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); @@ -3882,6 +3880,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3981,6 +3980,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PAR Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -4456,7 +4456,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS){ } OPCODE_DEFERRED_INC(COMPARE_OP); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } // _COMPARE_OP { @@ -7395,11 +7395,74 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS){ } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS){ + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD); + PREDICTED_JUMP_BACKWARD:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + /* Skip 1 cache entry */ + // _SPECIALIZE_JUMP_BACKWARD + { + #if ENABLE_SPECIALIZATION + if (this_instr->op.code == JUMP_BACKWARD) { + this_instr->op.code = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; + // Need to re-dispatch so the warmup counter isn't off by one: + next_instr = this_instr; + DISPATCH_SAME_OPARG(); + } + #endif + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + } + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD); + INSTRUCTION_STATS(JUMP_BACKWARD_JIT); + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ // _CHECK_PERIODIC { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); @@ -7411,16 +7474,21 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS){ if (err != 0) goto error; } } - // _JUMP_BACKWARD + // _JUMP_BACKWARD_NO_INTERRUPT { - uint16_t the_counter = read_u16(&this_instr[1].cache); - (void)the_counter; + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); + } + // _JIT + { #ifdef _Py_TIER2 - #if ENABLE_SPECIALIZATION _Py_BackoffCounter counter = this_instr[1].counter; - if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) { + if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { _Py_CODEUNIT *start = this_instr; /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ while (oparg > 255) { @@ -7447,8 +7515,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS){ else { ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); } - #endif /* ENABLE_SPECIALIZATION */ - #endif /* _Py_TIER2 */ + #endif } } DISPATCH(); @@ -7480,6 +7547,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_ * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ + assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); } DISPATCH(); @@ -7501,6 +7569,54 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_ TAIL_CALL(resume_with_error); } +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS){ + { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + } + } + DISPATCH(); + pop_4_error: + TAIL_CALL(pop_4_error); + pop_3_error: + TAIL_CALL(pop_3_error); + pop_2_error: + TAIL_CALL(pop_2_error); + pop_1_error: + TAIL_CALL(pop_1_error); + error: + TAIL_CALL(error); + exception_unwind: + TAIL_CALL(exception_unwind); + exit_unwind: + TAIL_CALL(exit_unwind); + resume_with_error: + TAIL_CALL(resume_with_error); +} + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS){ { frame->instr_ptr = next_instr; @@ -7626,7 +7742,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ (void)this_instr; _PyStackRef owner; _PyStackRef attr; - _PyStackRef self_or_null = PyStackRef_NULL; + _PyStackRef *self_or_null; // _SPECIALIZE_LOAD_ATTR { owner = stack_pointer[-1]; @@ -7648,6 +7764,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ /* Skip 8 cache entries */ // _LOAD_ATTR { + self_or_null = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); PyObject *attr_o; if (oparg & 1) { @@ -7662,7 +7779,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ meth | self | arg1 | ... | argN */ assert(attr_o != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership + self_or_null[0] = owner; // Transfer ownership } else { /* meth is not an unbound method (but a regular attr, or @@ -7673,7 +7790,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ */ PyStackRef_CLOSE(owner); if (attr_o == NULL) goto pop_1_error; - self_or_null = PyStackRef_NULL; + self_or_null[0] = PyStackRef_NULL; } } else { @@ -7683,14 +7800,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); if (attr_o == NULL) goto pop_1_error; - /* We need to define self_or_null on all paths */ - self_or_null = PyStackRef_NULL; } attr = PyStackRef_FromPyObjectSteal(attr_o); } stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = self_or_null; - stack_pointer += (oparg & 1); + stack_pointer += (oparg&1); assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -7715,6 +7829,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -7738,9 +7853,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -7768,6 +7886,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -7797,9 +7916,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_C STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -7827,6 +7949,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_C Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -7881,6 +8004,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDE Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -7918,10 +8042,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -7949,12 +8076,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; + _PyStackRef self; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { @@ -8010,12 +8138,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; + _PyStackRef self; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { @@ -8064,12 +8193,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; + _PyStackRef self; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { @@ -8132,6 +8262,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAI Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -8171,10 +8302,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM attr = PyStackRef_FromPyObjectSteal(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -8202,6 +8336,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -8251,6 +8386,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT( Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -8313,6 +8449,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VAL Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -8396,6 +8533,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -8425,10 +8563,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -8456,6 +8597,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -8510,10 +8652,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); UNLOCK_OBJECT(dict); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -8633,11 +8778,21 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS){ * marshalling can intern strings and make them immortal. */ PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); value = PyStackRef_FromPyObjectNew(obj); - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT + #ifdef Py_GIL_DISABLED + uint8_t expected = LOAD_CONST; + if (!_Py_atomic_compare_exchange_uint8( + &this_instr->op.code, &expected, + _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL)) { + // We might lose a race with instrumentation, which we don't care about. + assert(expected >= MIN_INSTRUMENTED_OPCODE); + } + #else if (this_instr->op.code == LOAD_CONST) { this_instr->op.code = _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL; } #endif + #endif stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -9081,6 +9236,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS){ _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*res)) goto error; + } + // _PUSH_NULL_CONDITIONAL + { null = PyStackRef_NULL; } if (oparg & 1) stack_pointer[1] = null; @@ -9109,6 +9267,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); @@ -9149,6 +9308,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA res = PyStackRef_FromPyObjectSteal(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); + } + // _PUSH_NULL_CONDITIONAL + { null = PyStackRef_NULL; } stack_pointer[0] = res; @@ -9178,6 +9340,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); @@ -9210,6 +9373,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PAR res = PyStackRef_FromPyObjectSteal(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); + } + // _PUSH_NULL_CONDITIONAL + { null = PyStackRef_NULL; } stack_pointer[0] = res; @@ -9487,6 +9653,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) goto error; attr = PyStackRef_FromPyObjectSteal(attr_o); + } + // _PUSH_NULL_CONDITIONAL + { null = PyStackRef_NULL; } stack_pointer[0] = attr; @@ -11069,6 +11238,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); @@ -11145,6 +11315,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_C Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); @@ -11200,6 +11371,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); @@ -11828,6 +12000,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS){ { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); @@ -12657,7 +12830,9 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [INTERPRETER_EXIT] = _TAIL_CALL_INTERPRETER_EXIT, [IS_OP] = _TAIL_CALL_IS_OP, [JUMP_BACKWARD] = _TAIL_CALL_JUMP_BACKWARD, + [JUMP_BACKWARD_JIT] = _TAIL_CALL_JUMP_BACKWARD_JIT, [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT, + [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_JUMP_BACKWARD_NO_JIT, [JUMP_FORWARD] = _TAIL_CALL_JUMP_FORWARD, [LIST_APPEND] = _TAIL_CALL_LIST_APPEND, [LIST_EXTEND] = _TAIL_CALL_LIST_EXTEND, @@ -12791,8 +12966,6 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [146] = _TAIL_CALL_UNKNOWN_OPCODE, [147] = _TAIL_CALL_UNKNOWN_OPCODE, [148] = _TAIL_CALL_UNKNOWN_OPCODE, - [230] = _TAIL_CALL_UNKNOWN_OPCODE, - [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, [233] = _TAIL_CALL_UNKNOWN_OPCODE, [234] = _TAIL_CALL_UNKNOWN_OPCODE, From 7e1b8a5257258daf96121ddeea0b642b4ca372cb Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 30 Jan 2025 20:14:33 +0800 Subject: [PATCH 083/110] Address review partially --- ...-01-10-18-56-20.gh-issue-128563.baDvls.rst | 2 +- Python/ceval_macros.h | 18 - Python/generated_cases.c.h | 1516 ++++++++++++++--- Python/generated_tail_call_handlers.c.h | 1516 ++++++++++++++--- Tools/cases_generator/generators_common.py | 15 +- Tools/cases_generator/tier1_generator.py | 3 + .../tier1_tail_call_generator.py | 30 + 7 files changed, 2514 insertions(+), 586 deletions(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst index 63d177053cae71..4d29f346cb6251 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst @@ -1 +1 @@ -A new type of interpreter has been added to CPython. This interpreter uses tail calls for its instruction handlers. Preliminary benchmark results suggest 7-11% geometric mean faster on pyperformance (depending on platform), and up to 30% faster on Python-intensive workloads. This interpreter currently only works on newer compilers, such as ``clang-19``. Other compilers will continue using the old interpreter. Patch by Ken Jin, with ideas by Mark Shannon, Garret Gu, Haoran Xu, and Josh Haberman. +A new type of interpreter has been added to CPython. This interpreter uses tail calls for its instruction handlers. Preliminary benchmark results suggest 7-11% geometric mean faster on pyperformance (depending on platform), and up to 30% faster on Python-intensive workloads. This interpreter currently only works on newer compilers, such as ``clang-19``. Other compilers will continue using the old interpreter. Patch by Ken Jin, with ideas on how to implement this in CPython by Mark Shannon, Garret Gu, Haoran Xu, and Josh Haberman. diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index df60caef49444c..1ffe31774e1951 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -282,24 +282,6 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define UPDATE_MISS_STATS(INSTNAME) ((void)0) #endif -#ifdef Py_TAIL_CALL_INTERP -# define GO_TO_INSTRUCTION_IF(COND, INSTNAME, SIZE) \ - if ((COND)) { \ - /* This is only a single jump on release builds! */ \ - UPDATE_MISS_STATS((INSTNAME)); \ - assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); \ - Py_MUSTTAIL \ - return (INSTRUCTION_TABLE[INSTNAME])(frame, stack_pointer, tstate, next_instr - 1 - SIZE, opcode, oparg); \ - } -#else -# define GO_TO_INSTRUCTION_IF(COND, INSTNAME, SIZE) \ - if ((COND)) { \ - /* This is only a single jump on release builds! */ \ - UPDATE_MISS_STATS((INSTNAME)); \ - assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); \ - goto PREDICTED_##INSTNAME; \ - } -#endif // Try to lock an object in the free threading build, if it's not already // locked. Use with a DEOPT_IF() to deopt if the object is already locked. diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 566394113e272d..6505d48f15d4c2 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -69,7 +69,8 @@ } TARGET(BINARY_OP_ADD_FLOAT) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -82,8 +83,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + goto PREDICTED_BINARY_OP; + } + if (!PyFloat_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + goto PREDICTED_BINARY_OP; + } } /* Skip 5 cache entries */ // _BINARY_OP_ADD_FLOAT @@ -107,7 +116,8 @@ } TARGET(BINARY_OP_ADD_INT) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -120,8 +130,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + if (!PyLong_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + goto PREDICTED_BINARY_OP; + } + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + goto PREDICTED_BINARY_OP; + } } /* Skip 5 cache entries */ // _BINARY_OP_ADD_INT @@ -144,7 +162,8 @@ } TARGET(BINARY_OP_ADD_UNICODE) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -157,8 +176,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + if (!PyUnicode_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + goto PREDICTED_BINARY_OP; + } + if (!PyUnicode_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + goto PREDICTED_BINARY_OP; + } } /* Skip 5 cache entries */ // _BINARY_OP_ADD_UNICODE @@ -203,7 +230,11 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int res = d->guard(left_o, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); - GO_TO_INSTRUCTION_IF(!res, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + if (!res) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + goto PREDICTED_BINARY_OP; + } } /* Skip -4 cache entry */ // _BINARY_OP_EXTEND @@ -228,7 +259,8 @@ } TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -240,8 +272,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + if (!PyUnicode_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + goto PREDICTED_BINARY_OP; + } + if (!PyUnicode_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + goto PREDICTED_BINARY_OP; + } } /* Skip 5 cache entries */ // _BINARY_OP_INPLACE_ADD_UNICODE @@ -258,7 +298,11 @@ next_oparg = CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); - GO_TO_INSTRUCTION_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + goto PREDICTED_BINARY_OP; + } STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -291,7 +335,8 @@ } TARGET(BINARY_OP_MULTIPLY_FLOAT) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -304,8 +349,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + goto PREDICTED_BINARY_OP; + } + if (!PyFloat_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + goto PREDICTED_BINARY_OP; + } } /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_FLOAT @@ -329,7 +382,8 @@ } TARGET(BINARY_OP_MULTIPLY_INT) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -342,8 +396,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + if (!PyLong_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + goto PREDICTED_BINARY_OP; + } + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + goto PREDICTED_BINARY_OP; + } } /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_INT @@ -366,7 +428,8 @@ } TARGET(BINARY_OP_SUBTRACT_FLOAT) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -379,8 +442,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + goto PREDICTED_BINARY_OP; + } + if (!PyFloat_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + goto PREDICTED_BINARY_OP; + } } /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_FLOAT @@ -404,7 +475,8 @@ } TARGET(BINARY_OP_SUBTRACT_INT) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -417,8 +489,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + if (!PyLong_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + goto PREDICTED_BINARY_OP; + } + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + goto PREDICTED_BINARY_OP; + } } /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_INT @@ -538,7 +618,8 @@ } TARGET(BINARY_SUBSCR_DICT) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); @@ -550,7 +631,11 @@ dict_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + goto PREDICTED_BINARY_SUBSCR; + } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -573,7 +658,8 @@ } TARGET(BINARY_SUBSCR_GETITEM) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); @@ -584,22 +670,42 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + goto PREDICTED_BINARY_SUBSCR; + } } // _BINARY_SUBSCR_CHECK_FUNC { container = stack_pointer[-2]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - GO_TO_INSTRUCTION_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + goto PREDICTED_BINARY_SUBSCR; + } PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - GO_TO_INSTRUCTION_IF(getitem_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (getitem_o == NULL) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + goto PREDICTED_BINARY_SUBSCR; + } assert(PyFunction_Check(getitem_o)); uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - GO_TO_INSTRUCTION_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + goto PREDICTED_BINARY_SUBSCR; + } PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); - GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + goto PREDICTED_BINARY_SUBSCR; + } getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); } @@ -632,7 +738,8 @@ } TARGET(BINARY_SUBSCR_LIST_INT) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); @@ -644,19 +751,39 @@ list_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - GO_TO_INSTRUCTION_IF(!PyList_CheckExact(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (!PyLong_CheckExact(sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + goto PREDICTED_BINARY_SUBSCR; + } + if (!PyList_CheckExact(list)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + goto PREDICTED_BINARY_SUBSCR; + } // Deopt unless 0 <= sub < PyList_Size(list) - GO_TO_INSTRUCTION_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + goto PREDICTED_BINARY_SUBSCR; + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; #ifdef Py_GIL_DISABLED _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); stack_pointer = _PyFrame_GetStackPointer(frame); - GO_TO_INSTRUCTION_IF(res_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (res_o == NULL) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + goto PREDICTED_BINARY_SUBSCR; + } STAT_INC(BINARY_SUBSCR, hit); #else - GO_TO_INSTRUCTION_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (index >= PyList_GET_SIZE(list)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + goto PREDICTED_BINARY_SUBSCR; + } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); @@ -672,7 +799,8 @@ } TARGET(BINARY_SUBSCR_STR_INT) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); @@ -684,14 +812,34 @@ str_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - GO_TO_INSTRUCTION_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (!PyLong_CheckExact(sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + goto PREDICTED_BINARY_SUBSCR; + } + if (!PyUnicode_CheckExact(str)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + goto PREDICTED_BINARY_SUBSCR; + } + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + goto PREDICTED_BINARY_SUBSCR; + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - GO_TO_INSTRUCTION_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (PyUnicode_GET_LENGTH(str) <= index) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + goto PREDICTED_BINARY_SUBSCR; + } // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - GO_TO_INSTRUCTION_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + goto PREDICTED_BINARY_SUBSCR; + } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); @@ -704,7 +852,8 @@ } TARGET(BINARY_SUBSCR_TUPLE_INT) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); @@ -716,12 +865,28 @@ tuple_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - GO_TO_INSTRUCTION_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (!PyLong_CheckExact(sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + goto PREDICTED_BINARY_SUBSCR; + } + if (!PyTuple_CheckExact(tuple)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + goto PREDICTED_BINARY_SUBSCR; + } // Deopt unless 0 <= sub < PyTuple_Size(list) - GO_TO_INSTRUCTION_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + goto PREDICTED_BINARY_SUBSCR; + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - GO_TO_INSTRUCTION_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (index >= PyTuple_GET_SIZE(tuple)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + goto PREDICTED_BINARY_SUBSCR; + } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); @@ -1104,7 +1269,11 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } } // _CHECK_AND_ALLOCATE_OBJECT { @@ -1115,17 +1284,33 @@ self = &stack_pointer[-1 - oparg]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyStackRef_IsNull(null[0])) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } + if (!PyType_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } PyTypeObject *tp = (PyTypeObject *)callable_o; - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL, INLINE_CACHE_ENTRIES_CALL); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } assert(tp->tp_new == PyBaseObject_Type.tp_new); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); assert(tp->tp_alloc == PyType_GenericAlloc); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; - GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *self_o = PyType_GenericAlloc(tp, 0); @@ -1203,14 +1388,26 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS { null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyStackRef_IsNull(null[0])) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { @@ -1229,9 +1426,17 @@ callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - GO_TO_INSTRUCTION_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } PyFunctionObject *func = (PyFunctionObject *)callable_o; - GO_TO_INSTRUCTION_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } } // _CHECK_FUNCTION_EXACT_ARGS { @@ -1240,15 +1445,27 @@ assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - GO_TO_INSTRUCTION_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } } // _INIT_CALL_PY_EXACT_ARGS { @@ -1307,7 +1524,11 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } } // _CHECK_METHOD_VERSION { @@ -1315,11 +1536,27 @@ callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - GO_TO_INSTRUCTION_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } PyObject *func = ((PyMethodObject *)callable_o)->im_func; - GO_TO_INSTRUCTION_IF(!PyFunction_Check(func), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(((PyFunctionObject *)func)->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } + if (!PyStackRef_IsNull(null[0])) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } } // _EXPAND_METHOD { @@ -1392,7 +1629,8 @@ } TARGET(CALL_BUILTIN_CLASS) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_CLASS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -1408,7 +1646,11 @@ self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - GO_TO_INSTRUCTION_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyType_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } PyTypeObject *tp = (PyTypeObject *)callable_o; int total_args = oparg; _PyStackRef *arguments = args; @@ -1416,7 +1658,11 @@ arguments--; total_args++; } - GO_TO_INSTRUCTION_IF(tp->tp_vectorcall == NULL, CALL, INLINE_CACHE_ENTRIES_CALL); + if (tp->tp_vectorcall == NULL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { @@ -1470,7 +1716,8 @@ } TARGET(CALL_BUILTIN_FAST) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -1493,8 +1740,16 @@ arguments--; total_args++; } - GO_TO_INSTRUCTION_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ @@ -1554,7 +1809,8 @@ } TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -1577,8 +1833,16 @@ arguments--; total_args++; } - GO_TO_INSTRUCTION_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } + if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } STAT_INC(CALL, hit); /* res = func(self, arguments, nargs, kwnames) */ _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1639,7 +1903,8 @@ } TARGET(CALL_BUILTIN_O) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -1661,11 +1926,27 @@ args--; total_args++; } - GO_TO_INSTRUCTION_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } // CPython promises to check all non-vectorcall function calls. - GO_TO_INSTRUCTION_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); + if (tstate->c_recursion_remaining <= 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; @@ -1913,7 +2194,8 @@ } TARGET(CALL_ISINSTANCE) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ISINSTANCE); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -1934,9 +2216,17 @@ arguments--; total_args++; } - GO_TO_INSTRUCTION_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); + if (total_args != 2) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } PyInterpreterState *interp = tstate->interp; - GO_TO_INSTRUCTION_IF(callable_o != interp->callable_cache.isinstance, CALL, INLINE_CACHE_ENTRIES_CALL); + if (callable_o != interp->callable_cache.isinstance) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } STAT_INC(CALL, hit); _PyStackRef cls_stackref = arguments[1]; _PyStackRef inst_stackref = arguments[0]; @@ -2134,7 +2424,11 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + goto PREDICTED_CALL_KW; + } } // _CHECK_METHOD_VERSION_KW { @@ -2142,11 +2436,27 @@ callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - GO_TO_INSTRUCTION_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + goto PREDICTED_CALL_KW; + } PyObject *func = ((PyMethodObject *)callable_o)->im_func; - GO_TO_INSTRUCTION_IF(!PyFunction_Check(func), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); - GO_TO_INSTRUCTION_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); - GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null[0]), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + goto PREDICTED_CALL_KW; + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + goto PREDICTED_CALL_KW; + } + if (!PyStackRef_IsNull(null[0])) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + goto PREDICTED_CALL_KW; + } } // _EXPAND_METHOD_KW { @@ -2225,7 +2535,8 @@ } TARGET(CALL_KW_NON_PY) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_NON_PY); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); @@ -2240,8 +2551,16 @@ { callable = &stack_pointer[-3 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - GO_TO_INSTRUCTION_IF(PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); - GO_TO_INSTRUCTION_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + if (PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + goto PREDICTED_CALL_KW; + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + goto PREDICTED_CALL_KW; + } } // _CALL_KW_NON_PY { @@ -2332,16 +2651,28 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + goto PREDICTED_CALL_KW; + } } // _CHECK_FUNCTION_VERSION_KW { callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - GO_TO_INSTRUCTION_IF(!PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + goto PREDICTED_CALL_KW; + } PyFunctionObject *func = (PyFunctionObject *)callable_o; - GO_TO_INSTRUCTION_IF(func->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + goto PREDICTED_CALL_KW; + } } // _PY_FRAME_KW { @@ -2405,7 +2736,8 @@ } TARGET(CALL_LEN) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LEN); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -2425,9 +2757,17 @@ args--; total_args++; } - GO_TO_INSTRUCTION_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } PyInterpreterState *interp = tstate->interp; - GO_TO_INSTRUCTION_IF(callable_o != interp->callable_cache.len, CALL, INLINE_CACHE_ENTRIES_CALL); + if (callable_o != interp->callable_cache.len) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } STAT_INC(CALL, hit); _PyStackRef arg_stackref = args[0]; PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); @@ -2452,7 +2792,8 @@ } TARGET(CALL_LIST_APPEND) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LIST_APPEND); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -2468,10 +2809,22 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); PyInterpreterState *interp = tstate->interp; - GO_TO_INSTRUCTION_IF(callable_o != interp->callable_cache.list_append, CALL, INLINE_CACHE_ENTRIES_CALL); + if (callable_o != interp->callable_cache.list_append) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } assert(self_o != NULL); - GO_TO_INSTRUCTION_IF(!PyList_Check(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyList_Check(self_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } + if (!LOCK_OBJECT(self_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); @@ -2490,7 +2843,8 @@ } TARGET(CALL_METHOD_DESCRIPTOR_FAST) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -2514,11 +2868,23 @@ } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ - GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } PyMethodDef *meth = method->d_method; - GO_TO_INSTRUCTION_IF(meth->ml_flags != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); + if (meth->ml_flags != METH_FASTCALL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -2576,7 +2942,8 @@ } TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -2599,12 +2966,24 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } PyMethodDef *meth = method->d_method; - GO_TO_INSTRUCTION_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); + if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } PyTypeObject *d_type = method->d_common.d_type; PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(self, d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!Py_IS_TYPE(self, d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -2662,7 +3041,8 @@ } TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -2684,16 +3064,36 @@ args--; total_args++; } - GO_TO_INSTRUCTION_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } PyMethodDef *meth = method->d_method; _PyStackRef self_stackref = args[0]; PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(meth->ml_flags != METH_NOARGS, CALL, INLINE_CACHE_ENTRIES_CALL); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } + if (meth->ml_flags != METH_NOARGS) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } // CPython promises to check all non-vectorcall function calls. - GO_TO_INSTRUCTION_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); + if (tstate->c_recursion_remaining <= 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -2734,7 +3134,8 @@ } TARGET(CALL_METHOD_DESCRIPTOR_O) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -2756,16 +3157,36 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - GO_TO_INSTRUCTION_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + if (total_args != 2) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } PyMethodDef *meth = method->d_method; - GO_TO_INSTRUCTION_IF(meth->ml_flags != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); + if (meth->ml_flags != METH_O) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } // CPython promises to check all non-vectorcall function calls. - GO_TO_INSTRUCTION_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); + if (tstate->c_recursion_remaining <= 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } _PyStackRef arg_stackref = args[1]; _PyStackRef self_stackref = args[0]; - GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -2809,7 +3230,8 @@ } TARGET(CALL_NON_PY_GENERAL) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_NON_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -2823,8 +3245,16 @@ { callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - GO_TO_INSTRUCTION_IF(PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + if (PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } } // _CALL_NON_PY_GENERAL { @@ -2909,16 +3339,28 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - GO_TO_INSTRUCTION_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } PyFunctionObject *func = (PyFunctionObject *)callable_o; - GO_TO_INSTRUCTION_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } } // _CHECK_FUNCTION_EXACT_ARGS { @@ -2927,15 +3369,27 @@ assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - GO_TO_INSTRUCTION_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } } // _INIT_CALL_PY_EXACT_ARGS { @@ -2991,16 +3445,28 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - GO_TO_INSTRUCTION_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } PyFunctionObject *func = (PyFunctionObject *)callable_o; - GO_TO_INSTRUCTION_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } } // _PY_FRAME_GENERAL { @@ -3058,7 +3524,8 @@ } TARGET(CALL_STR_1) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_STR_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3076,8 +3543,16 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } + if (callable_o != (PyObject *)&PyUnicode_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Str(arg_o); @@ -3109,7 +3584,8 @@ } TARGET(CALL_TUPLE_1) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TUPLE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3127,8 +3603,16 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(callable_o != (PyObject *)&PyTuple_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } + if (callable_o != (PyObject *)&PyTuple_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PySequence_Tuple(arg_o); @@ -3160,7 +3644,8 @@ } TARGET(CALL_TYPE_1) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TYPE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3176,8 +3661,16 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(callable_o != (PyObject *)&PyType_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } + if (callable_o != (PyObject *)&PyType_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + goto PREDICTED_CALL; + } STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); PyStackRef_CLOSE(arg); @@ -3366,7 +3859,8 @@ } TARGET(COMPARE_OP_FLOAT) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_FLOAT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); @@ -3379,8 +3873,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + goto PREDICTED_COMPARE_OP; + } + if (!PyFloat_CheckExact(right_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + goto PREDICTED_COMPARE_OP; + } } /* Skip 1 cache entry */ // _COMPARE_OP_FLOAT @@ -3404,7 +3906,8 @@ } TARGET(COMPARE_OP_INT) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_INT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); @@ -3417,16 +3920,32 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + if (!PyLong_CheckExact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + goto PREDICTED_COMPARE_OP; + } + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + goto PREDICTED_COMPARE_OP; + } } /* Skip 1 cache entry */ // _COMPARE_OP_INT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - GO_TO_INSTRUCTION_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + if (!_PyLong_IsCompact((PyLongObject *)left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + goto PREDICTED_COMPARE_OP; + } + if (!_PyLong_IsCompact((PyLongObject *)right_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + goto PREDICTED_COMPARE_OP; + } STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && _PyLong_DigitCount((PyLongObject *)right_o) <= 1); @@ -3446,7 +3965,8 @@ } TARGET(COMPARE_OP_STR) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_STR); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); @@ -3459,8 +3979,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + if (!PyUnicode_CheckExact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + goto PREDICTED_COMPARE_OP; + } + if (!PyUnicode_CheckExact(right_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + goto PREDICTED_COMPARE_OP; + } } /* Skip 1 cache entry */ // _COMPARE_OP_STR @@ -3531,7 +4059,8 @@ } TARGET(CONTAINS_OP_DICT) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_DICT); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); @@ -3543,7 +4072,11 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(right_o), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); + if (!PyDict_CheckExact(right_o)) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + goto PREDICTED_CONTAINS_OP; + } STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); @@ -3559,7 +4092,8 @@ } TARGET(CONTAINS_OP_SET) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_SET); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); @@ -3571,7 +4105,11 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); + if (!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o))) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + goto PREDICTED_CONTAINS_OP; + } STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4068,7 +4606,8 @@ } TARGET(FOR_ITER_GEN) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_GEN); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); @@ -4078,14 +4617,26 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + goto PREDICTED_FOR_ITER; + } } // _FOR_ITER_GEN_FRAME { iter = stack_pointer[-1]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - GO_TO_INSTRUCTION_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); - GO_TO_INSTRUCTION_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + if (Py_TYPE(gen) != &PyGen_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + goto PREDICTED_FOR_ITER; + } + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + goto PREDICTED_FOR_ITER; + } STAT_INC(FOR_ITER, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, PyStackRef_None); @@ -4116,7 +4667,8 @@ } TARGET(FOR_ITER_LIST) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_LIST); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); @@ -4126,7 +4678,11 @@ // _ITER_CHECK_LIST { iter = stack_pointer[-1]; - GO_TO_INSTRUCTION_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + goto PREDICTED_FOR_ITER; + } } // _ITER_JUMP_LIST { @@ -4167,7 +4723,8 @@ } TARGET(FOR_ITER_RANGE) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_RANGE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); @@ -4178,7 +4735,11 @@ { iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - GO_TO_INSTRUCTION_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + goto PREDICTED_FOR_ITER; + } } // _ITER_JUMP_RANGE { @@ -4210,7 +4771,8 @@ } TARGET(FOR_ITER_TUPLE) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_TUPLE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); @@ -4220,7 +4782,11 @@ // _ITER_CHECK_TUPLE { iter = stack_pointer[-1]; - GO_TO_INSTRUCTION_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + goto PREDICTED_FOR_ITER; + } } // _ITER_JUMP_TUPLE { @@ -5474,9 +6040,17 @@ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - GO_TO_INSTRUCTION_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (!PyType_Check(owner_o)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS @@ -5513,16 +6087,28 @@ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - GO_TO_INSTRUCTION_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (!PyType_Check(owner_o)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } // _GUARD_TYPE_VERSION { uint32_t type_version = read_u32(&this_instr[4].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } // _LOAD_ATTR_CLASS { @@ -5557,17 +6143,33 @@ PyObject *getattribute = read_obj(&this_instr[6].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert((oparg & 1) == 0); - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); - GO_TO_INSTRUCTION_IF(f->func_version != func_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (f->func_version != func_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 2); - GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( @@ -5596,14 +6198,22 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } // _CHECK_MANAGED_OBJECT_HAS_VALUES { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - GO_TO_INSTRUCTION_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } // _LOAD_ATTR_INSTANCE_VALUE { @@ -5611,10 +6221,18 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); - GO_TO_INSTRUCTION_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } #ifdef Py_GIL_DISABLED if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { - GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } #else attr = PyStackRef_FromPyObjectNew(attr_o); @@ -5650,7 +6268,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } // _CHECK_ATTR_METHOD_LAZY_DICT { @@ -5658,7 +6280,11 @@ char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); /* This object has a __dict__, just not yet created */ - GO_TO_INSTRUCTION_IF(dict != NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (dict != NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } /* Skip 1 cache entry */ // _LOAD_ATTR_METHOD_LAZY_DICT @@ -5694,7 +6320,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_NO_DICT @@ -5731,14 +6361,22 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyDictValues *ivs = _PyObject_InlineValues(owner_o); - GO_TO_INSTRUCTION_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } // _GUARD_KEYS_VERSION { @@ -5746,7 +6384,11 @@ PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } // _LOAD_ATTR_METHOD_WITH_VALUES { @@ -5782,11 +6424,19 @@ owner = stack_pointer[-1]; uint32_t dict_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - GO_TO_INSTRUCTION_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } mod_keys = keys; } // _LOAD_ATTR_MODULE_FROM_KEYS @@ -5797,11 +6447,19 @@ PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); // Clear mod_keys from stack in case we need to deopt - GO_TO_INSTRUCTION_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); if (!increfed) { - GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } #else Py_INCREF(attr_o); @@ -5837,7 +6495,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT @@ -5869,14 +6531,22 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyDictValues *ivs = _PyObject_InlineValues(owner_o); - GO_TO_INSTRUCTION_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } // _GUARD_KEYS_VERSION { @@ -5884,7 +6554,11 @@ PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { @@ -5910,7 +6584,11 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } // _GUARD_TYPE_VERSION { @@ -5918,7 +6596,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } /* Skip 2 cache entries */ // _LOAD_ATTR_PROPERTY_FRAME @@ -5928,10 +6610,26 @@ assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; PyCodeObject *code = (PyCodeObject *)f->func_code; - GO_TO_INSTRUCTION_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - GO_TO_INSTRUCTION_IF(code->co_kwonlyargcount, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - GO_TO_INSTRUCTION_IF(code->co_argcount != 1, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } + if (code->co_kwonlyargcount) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } + if (code->co_argcount != 1) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } STAT_INC(LOAD_ATTR, hit); new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); new_frame->localsplus[0] = owner; @@ -5981,7 +6679,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } // _LOAD_ATTR_SLOT { @@ -5989,10 +6691,18 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **addr = (PyObject **)((char *)owner_o + index); PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); - GO_TO_INSTRUCTION_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); - GO_TO_INSTRUCTION_IF(!increfed, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } #else attr = PyStackRef_FromPyObjectNew(attr_o); #endif @@ -6028,14 +6738,22 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } // _CHECK_ATTR_WITH_HINT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); - GO_TO_INSTRUCTION_IF(dict_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (dict_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } assert(PyDict_CheckExact((PyObject *)dict_o)); dict = dict_o; } @@ -6044,26 +6762,46 @@ uint16_t hint = read_u16(&this_instr[4].cache); PyObject *attr_o; if (!LOCK_OBJECT(dict)) { - GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } if (hint >= (size_t)dict->ma_keys->dk_nentries) { UNLOCK_OBJECT(dict); - GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); - GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } attr_o = ep->me_value; if (attr_o == NULL) { UNLOCK_OBJECT(dict); - GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + goto PREDICTED_LOAD_ATTR; + } } STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); @@ -6443,18 +7181,34 @@ { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + goto PREDICTED_LOAD_GLOBAL; + } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + goto PREDICTED_LOAD_GLOBAL; + } assert(DK_IS_UNICODE(keys)); } // _GUARD_BUILTINS_VERSION_PUSH_KEYS { uint16_t version = read_u16(&this_instr[3].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); - GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + goto PREDICTED_LOAD_GLOBAL; + } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + goto PREDICTED_LOAD_GLOBAL; + } builtins_keys = keys; assert(DK_IS_UNICODE(builtins_keys)); } @@ -6463,10 +7217,18 @@ uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - GO_TO_INSTRUCTION_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + if (res_o == NULL) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + goto PREDICTED_LOAD_GLOBAL; + } #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - GO_TO_INSTRUCTION_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + goto PREDICTED_LOAD_GLOBAL; + } #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -6498,9 +7260,17 @@ { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + goto PREDICTED_LOAD_GLOBAL; + } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + goto PREDICTED_LOAD_GLOBAL; + } globals_keys = keys; assert(DK_IS_UNICODE(globals_keys)); } @@ -6510,10 +7280,18 @@ uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - GO_TO_INSTRUCTION_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + if (res_o == NULL) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + goto PREDICTED_LOAD_GLOBAL; + } #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - GO_TO_INSTRUCTION_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + goto PREDICTED_LOAD_GLOBAL; + } #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -6722,7 +7500,8 @@ } TARGET(LOAD_SUPER_ATTR_ATTR) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); @@ -6738,8 +7517,16 @@ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(!(oparg & 1)); - GO_TO_INSTRUCTION_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); - GO_TO_INSTRUCTION_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + if (global_super != (PyObject *)&PySuper_Type) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + goto PREDICTED_LOAD_SUPER_ATTR; + } + if (!PyType_Check(class)) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + goto PREDICTED_LOAD_SUPER_ATTR; + } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6757,7 +7544,8 @@ } TARGET(LOAD_SUPER_ATTR_METHOD) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); @@ -6774,8 +7562,16 @@ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(oparg & 1); - GO_TO_INSTRUCTION_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); - GO_TO_INSTRUCTION_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + if (global_super != (PyObject *)&PySuper_Type) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + goto PREDICTED_LOAD_SUPER_ATTR; + } + if (!PyType_Check(class)) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + goto PREDICTED_LOAD_SUPER_ATTR; + } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -7296,21 +8092,34 @@ } TARGET(RESUME_CHECK) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(RESUME_CHECK); static_assert(0 == 0, "incorrect cache size"); #if defined(__EMSCRIPTEN__) - GO_TO_INSTRUCTION_IF(_Py_emscripten_signal_clock == 0, RESUME, 0); + if (_Py_emscripten_signal_clock == 0) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + goto PREDICTED_RESUME; + } _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); - GO_TO_INSTRUCTION_IF(eval_breaker != version, RESUME, 0); + if (eval_breaker != version) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + goto PREDICTED_RESUME; + } #ifdef Py_GIL_DISABLED - GO_TO_INSTRUCTION_IF(frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME, 0); + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + goto PREDICTED_RESUME; + } #endif DISPATCH(); } @@ -7467,7 +8276,8 @@ } TARGET(SEND_GEN) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(SEND_GEN); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); @@ -7478,15 +8288,27 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, SEND, INLINE_CACHE_ENTRIES_SEND); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + goto PREDICTED_SEND; + } } // _SEND_GEN_FRAME { v = stack_pointer[-1]; receiver = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - GO_TO_INSTRUCTION_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND, INLINE_CACHE_ENTRIES_SEND); - GO_TO_INSTRUCTION_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND, INLINE_CACHE_ENTRIES_SEND); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + goto PREDICTED_SEND; + } + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + goto PREDICTED_SEND; + } STAT_INC(SEND, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, v); @@ -7677,11 +8499,19 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + if (!LOCK_OBJECT(owner_o)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + goto PREDICTED_STORE_ATTR; + } PyTypeObject *tp = Py_TYPE(owner_o); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UNLOCK_OBJECT(owner_o); - GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + goto PREDICTED_STORE_ATTR; + } } } // _GUARD_DORV_NO_DICT @@ -7692,7 +8522,11 @@ if (_PyObject_GetManagedDict(owner_o) || !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UNLOCK_OBJECT(owner_o); - GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + goto PREDICTED_STORE_ATTR; + } } } // _STORE_ATTR_INSTANCE_VALUE @@ -7736,14 +8570,22 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + goto PREDICTED_STORE_ATTR; + } } // _STORE_ATTR_SLOT { value = stack_pointer[-2]; uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + if (!LOCK_OBJECT(owner_o)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + goto PREDICTED_STORE_ATTR; + } char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; @@ -7774,7 +8616,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + goto PREDICTED_STORE_ATTR; + } } // _STORE_ATTR_WITH_HINT { @@ -7783,12 +8629,24 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - GO_TO_INSTRUCTION_IF(dict == NULL, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); - GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(dict), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + if (dict == NULL) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + goto PREDICTED_STORE_ATTR; + } + if (!LOCK_OBJECT(dict)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + goto PREDICTED_STORE_ATTR; + } #ifdef Py_GIL_DISABLED if (dict != _PyObject_GetManagedDict(owner_o)) { UNLOCK_OBJECT(dict); - GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + goto PREDICTED_STORE_ATTR; + } } #endif assert(PyDict_CheckExact((PyObject *)dict)); @@ -7796,17 +8654,29 @@ if (hint >= (size_t)dict->ma_keys->dk_nentries || !DK_IS_UNICODE(dict->ma_keys)) { UNLOCK_OBJECT(dict); - GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + goto PREDICTED_STORE_ATTR; + } } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + goto PREDICTED_STORE_ATTR; + } } PyObject *old_value = ep->me_value; if (old_value == NULL) { UNLOCK_OBJECT(dict); - GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + goto PREDICTED_STORE_ATTR; + } } _PyFrame_SetStackPointer(frame, stack_pointer); _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); @@ -8030,7 +8900,8 @@ } TARGET(STORE_SUBSCR_DICT) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); @@ -8042,7 +8913,11 @@ dict_st = stack_pointer[-2]; value = stack_pointer[-3]; PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(dict), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + goto PREDICTED_STORE_SUBSCR; + } STAT_INC(STORE_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, @@ -8057,7 +8932,8 @@ } TARGET(STORE_SUBSCR_LIST_INT) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); @@ -8070,16 +8946,36 @@ value = stack_pointer[-3]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); - GO_TO_INSTRUCTION_IF(!PyList_CheckExact(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + if (!PyLong_CheckExact(sub)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + goto PREDICTED_STORE_SUBSCR; + } + if (!PyList_CheckExact(list)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + goto PREDICTED_STORE_SUBSCR; + } // Ensure nonnegative, zero-or-one-digit ints. - GO_TO_INSTRUCTION_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + goto PREDICTED_STORE_SUBSCR; + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + if (!LOCK_OBJECT(list)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + goto PREDICTED_STORE_SUBSCR; + } // Ensure index < len(list) if (index >= PyList_GET_SIZE(list)) { UNLOCK_OBJECT(list); - GO_TO_INSTRUCTION_IF(true, STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + if (true) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + goto PREDICTED_STORE_SUBSCR; + } } STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); @@ -8170,7 +9066,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + goto PREDICTED_TO_BOOL; + } } // _REPLACE_WITH_TRUE { @@ -8183,7 +9083,8 @@ } TARGET(TO_BOOL_BOOL) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_BOOL); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); @@ -8191,13 +9092,18 @@ /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; - GO_TO_INSTRUCTION_IF(!PyStackRef_BoolCheck(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + if (!PyStackRef_BoolCheck(value)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + goto PREDICTED_TO_BOOL; + } STAT_INC(TO_BOOL, hit); DISPATCH(); } TARGET(TO_BOOL_INT) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_INT); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); @@ -8207,7 +9113,11 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + if (!PyLong_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + goto PREDICTED_TO_BOOL; + } STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value_o)) { assert(_Py_IsImmortal(value_o)); @@ -8222,7 +9132,8 @@ } TARGET(TO_BOOL_LIST) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_LIST); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); @@ -8232,7 +9143,11 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - GO_TO_INSTRUCTION_IF(!PyList_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + if (!PyList_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + goto PREDICTED_TO_BOOL; + } STAT_INC(TO_BOOL, hit); res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; PyStackRef_CLOSE(value); @@ -8241,7 +9156,8 @@ } TARGET(TO_BOOL_NONE) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_NONE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); @@ -8251,7 +9167,11 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - GO_TO_INSTRUCTION_IF(!PyStackRef_IsNone(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + if (!PyStackRef_IsNone(value)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + goto PREDICTED_TO_BOOL; + } STAT_INC(TO_BOOL, hit); res = PyStackRef_False; stack_pointer[-1] = res; @@ -8259,7 +9179,8 @@ } TARGET(TO_BOOL_STR) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_STR); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); @@ -8269,7 +9190,11 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + goto PREDICTED_TO_BOOL; + } STAT_INC(TO_BOOL, hit); if (value_o == &_Py_STR(empty)) { assert(_Py_IsImmortal(value_o)); @@ -8395,7 +9320,8 @@ } TARGET(UNPACK_SEQUENCE_LIST) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); @@ -8405,11 +9331,23 @@ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - GO_TO_INSTRUCTION_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); - GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + if (!PyList_CheckExact(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + goto PREDICTED_UNPACK_SEQUENCE; + } + if (!LOCK_OBJECT(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + goto PREDICTED_UNPACK_SEQUENCE; + } if (PyList_GET_SIZE(seq_o) != oparg) { UNLOCK_OBJECT(seq_o); - GO_TO_INSTRUCTION_IF(true, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + if (true) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + goto PREDICTED_UNPACK_SEQUENCE; + } } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq_o); @@ -8424,7 +9362,8 @@ } TARGET(UNPACK_SEQUENCE_TUPLE) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); @@ -8434,8 +9373,16 @@ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - GO_TO_INSTRUCTION_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); - GO_TO_INSTRUCTION_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + if (!PyTuple_CheckExact(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + goto PREDICTED_UNPACK_SEQUENCE; + } + if (PyTuple_GET_SIZE(seq_o) != oparg) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + goto PREDICTED_UNPACK_SEQUENCE; + } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { @@ -8448,7 +9395,8 @@ } TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); @@ -8459,8 +9407,16 @@ seq = stack_pointer[-1]; assert(oparg == 2); PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - GO_TO_INSTRUCTION_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); - GO_TO_INSTRUCTION_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + if (!PyTuple_CheckExact(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + goto PREDICTED_UNPACK_SEQUENCE; + } + if (PyTuple_GET_SIZE(seq_o) != 2) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + goto PREDICTED_UNPACK_SEQUENCE; + } STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 7c06ec7a6f37d0..f616a59aee313b 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -223,7 +223,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -236,8 +237,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PA left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!PyFloat_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } /* Skip 5 cache entries */ // _BINARY_OP_ADD_FLOAT @@ -279,7 +288,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -292,8 +302,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARA left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + if (!PyLong_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } /* Skip 5 cache entries */ // _BINARY_OP_ADD_INT @@ -334,7 +352,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -347,8 +366,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + if (!PyUnicode_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!PyUnicode_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } /* Skip 5 cache entries */ // _BINARY_OP_ADD_UNICODE @@ -411,7 +438,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAM _PyFrame_SetStackPointer(frame, stack_pointer); int res = d->guard(left_o, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); - GO_TO_INSTRUCTION_IF(!res, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + if (!res) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } /* Skip -4 cache entry */ // _BINARY_OP_EXTEND @@ -454,7 +485,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -466,8 +498,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TA left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + if (!PyUnicode_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!PyUnicode_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } /* Skip 5 cache entries */ // _BINARY_OP_INPLACE_ADD_UNICODE @@ -484,7 +524,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TA next_oparg = CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); - GO_TO_INSTRUCTION_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -535,7 +579,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -548,8 +593,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CA left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!PyFloat_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_FLOAT @@ -591,7 +644,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -604,8 +658,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + if (!PyLong_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_INT @@ -646,7 +708,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -659,8 +722,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CA left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!PyFloat_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_FLOAT @@ -702,7 +773,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -715,8 +787,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(left_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(right_o), BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP); + if (!PyLong_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_INT @@ -890,7 +970,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); @@ -902,7 +983,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PAR dict_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -943,7 +1028,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PAR Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); @@ -954,22 +1040,42 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _BINARY_SUBSCR_CHECK_FUNC { container = stack_pointer[-2]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - GO_TO_INSTRUCTION_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - GO_TO_INSTRUCTION_IF(getitem_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (getitem_o == NULL) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } assert(PyFunction_Check(getitem_o)); uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - GO_TO_INSTRUCTION_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); - GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); } @@ -1020,7 +1126,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); @@ -1032,19 +1139,39 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL list_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - GO_TO_INSTRUCTION_IF(!PyList_CheckExact(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (!PyLong_CheckExact(sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!PyList_CheckExact(list)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } // Deopt unless 0 <= sub < PyList_Size(list) - GO_TO_INSTRUCTION_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; #ifdef Py_GIL_DISABLED _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); stack_pointer = _PyFrame_GetStackPointer(frame); - GO_TO_INSTRUCTION_IF(res_o == NULL, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (res_o == NULL) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(BINARY_SUBSCR, hit); #else - GO_TO_INSTRUCTION_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (index >= PyList_GET_SIZE(list)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); @@ -1078,7 +1205,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); @@ -1090,14 +1218,34 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_ str_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - GO_TO_INSTRUCTION_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (!PyLong_CheckExact(sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!PyUnicode_CheckExact(str)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - GO_TO_INSTRUCTION_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (PyUnicode_GET_LENGTH(str) <= index) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - GO_TO_INSTRUCTION_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); @@ -1128,7 +1276,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); @@ -1140,12 +1289,28 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CAL tuple_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - GO_TO_INSTRUCTION_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (!PyLong_CheckExact(sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!PyTuple_CheckExact(tuple)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } // Deopt unless 0 <= sub < PyTuple_Size(list) - GO_TO_INSTRUCTION_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - GO_TO_INSTRUCTION_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + if (index >= PyTuple_GET_SIZE(tuple)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); @@ -1690,7 +1855,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_C /* Skip 1 cache entry */ // _CHECK_PEP_523 { - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _CHECK_AND_ALLOCATE_OBJECT { @@ -1701,17 +1870,33 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_C self = &stack_pointer[-1 - oparg]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyStackRef_IsNull(null[0])) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!PyType_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyTypeObject *tp = (PyTypeObject *)callable_o; - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL, INLINE_CACHE_ENTRIES_CALL); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } assert(tp->tp_new == PyBaseObject_Type.tp_new); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); assert(tp->tp_alloc == PyType_GenericAlloc); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; - GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *self_o = PyType_GenericAlloc(tp, 0); @@ -1807,14 +1992,26 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI /* Skip 1 cache entry */ // _CHECK_PEP_523 { - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS { null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyStackRef_IsNull(null[0])) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { @@ -1833,9 +2030,17 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - GO_TO_INSTRUCTION_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyFunctionObject *func = (PyFunctionObject *)callable_o; - GO_TO_INSTRUCTION_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _CHECK_FUNCTION_EXACT_ARGS { @@ -1844,15 +2049,27 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - GO_TO_INSTRUCTION_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _INIT_CALL_PY_EXACT_ARGS { @@ -1929,7 +2146,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_C /* Skip 1 cache entry */ // _CHECK_PEP_523 { - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _CHECK_METHOD_VERSION { @@ -1937,11 +2158,27 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_C callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - GO_TO_INSTRUCTION_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyObject *func = ((PyMethodObject *)callable_o)->im_func; - GO_TO_INSTRUCTION_IF(!PyFunction_Check(func), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(((PyFunctionObject *)func)->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null[0]), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!PyStackRef_IsNull(null[0])) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _EXPAND_METHOD { @@ -2032,7 +2269,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_C Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_CLASS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -2048,7 +2286,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - GO_TO_INSTRUCTION_IF(!PyType_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyType_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyTypeObject *tp = (PyTypeObject *)callable_o; int total_args = oparg; _PyStackRef *arguments = args; @@ -2056,7 +2298,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR arguments--; total_args++; } - GO_TO_INSTRUCTION_IF(tp->tp_vectorcall == NULL, CALL, INLINE_CACHE_ENTRIES_CALL); + if (tp->tp_vectorcall == NULL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { @@ -2128,7 +2374,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -2151,8 +2398,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARA arguments--; total_args++; } - GO_TO_INSTRUCTION_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ @@ -2230,7 +2485,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -2253,8 +2509,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS( arguments--; total_args++; } - GO_TO_INSTRUCTION_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(CALL, hit); /* res = func(self, arguments, nargs, kwnames) */ _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2333,7 +2597,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS( Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -2355,11 +2620,27 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS) args--; total_args++; } - GO_TO_INSTRUCTION_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(!PyCFunction_CheckExact(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } // CPython promises to check all non-vectorcall function calls. - GO_TO_INSTRUCTION_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); + if (tstate->c_recursion_remaining <= 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; @@ -2679,7 +2960,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ISINSTANCE); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -2700,9 +2982,17 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS arguments--; total_args++; } - GO_TO_INSTRUCTION_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); + if (total_args != 2) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyInterpreterState *interp = tstate->interp; - GO_TO_INSTRUCTION_IF(callable_o != interp->callable_cache.isinstance, CALL, INLINE_CACHE_ENTRIES_CALL); + if (callable_o != interp->callable_cache.isinstance) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(CALL, hit); _PyStackRef cls_stackref = arguments[1]; _PyStackRef inst_stackref = arguments[0]; @@ -2936,7 +3226,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_P /* Skip 1 cache entry */ // _CHECK_PEP_523 { - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _CHECK_METHOD_VERSION_KW { @@ -2944,11 +3238,27 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_P callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - GO_TO_INSTRUCTION_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyObject *func = ((PyMethodObject *)callable_o)->im_func; - GO_TO_INSTRUCTION_IF(!PyFunction_Check(func), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); - GO_TO_INSTRUCTION_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); - GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null[0]), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!PyStackRef_IsNull(null[0])) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _EXPAND_METHOD_KW { @@ -3045,7 +3355,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_P Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_NON_PY); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); @@ -3060,8 +3371,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) { callable = &stack_pointer[-3 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - GO_TO_INSTRUCTION_IF(PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); - GO_TO_INSTRUCTION_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + if (PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _CALL_KW_NON_PY { @@ -3170,16 +3489,28 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _CHECK_FUNCTION_VERSION_KW { callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - GO_TO_INSTRUCTION_IF(!PyFunction_Check(callable_o), CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyFunctionObject *func = (PyFunctionObject *)callable_o; - GO_TO_INSTRUCTION_IF(func->func_version != func_version, CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW); + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _PY_FRAME_KW { @@ -3261,7 +3592,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LEN); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3281,9 +3613,17 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ args--; total_args++; } - GO_TO_INSTRUCTION_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyInterpreterState *interp = tstate->interp; - GO_TO_INSTRUCTION_IF(callable_o != interp->callable_cache.len, CALL, INLINE_CACHE_ENTRIES_CALL); + if (callable_o != interp->callable_cache.len) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(CALL, hit); _PyStackRef arg_stackref = args[0]; PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); @@ -3326,7 +3666,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LIST_APPEND); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3342,10 +3683,22 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAM PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); PyInterpreterState *interp = tstate->interp; - GO_TO_INSTRUCTION_IF(callable_o != interp->callable_cache.list_append, CALL, INLINE_CACHE_ENTRIES_CALL); + if (callable_o != interp->callable_cache.list_append) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } assert(self_o != NULL); - GO_TO_INSTRUCTION_IF(!PyList_Check(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(self_o), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyList_Check(self_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!LOCK_OBJECT(self_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); @@ -3382,7 +3735,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3406,11 +3760,23 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ - GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyMethodDef *meth = method->d_method; - GO_TO_INSTRUCTION_IF(meth->ml_flags != METH_FASTCALL, CALL, INLINE_CACHE_ENTRIES_CALL); + if (meth->ml_flags != METH_FASTCALL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -3486,7 +3852,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3509,12 +3876,24 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyMethodDef *meth = method->d_method; - GO_TO_INSTRUCTION_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL, INLINE_CACHE_ENTRIES_CALL); + if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyTypeObject *d_type = method->d_common.d_type; PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(self, d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!Py_IS_TYPE(self, d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -3590,7 +3969,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3612,16 +3992,36 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TA args--; total_args++; } - GO_TO_INSTRUCTION_IF(total_args != 1, CALL, INLINE_CACHE_ENTRIES_CALL); + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyMethodDef *meth = method->d_method; _PyStackRef self_stackref = args[0]; PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(meth->ml_flags != METH_NOARGS, CALL, INLINE_CACHE_ENTRIES_CALL); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (meth->ml_flags != METH_NOARGS) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } // CPython promises to check all non-vectorcall function calls. - GO_TO_INSTRUCTION_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); + if (tstate->c_recursion_remaining <= 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -3680,7 +4080,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3702,16 +4103,36 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CA total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - GO_TO_INSTRUCTION_IF(total_args != 2, CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL, INLINE_CACHE_ENTRIES_CALL); + if (total_args != 2) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyMethodDef *meth = method->d_method; - GO_TO_INSTRUCTION_IF(meth->ml_flags != METH_O, CALL, INLINE_CACHE_ENTRIES_CALL); + if (meth->ml_flags != METH_O) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } // CPython promises to check all non-vectorcall function calls. - GO_TO_INSTRUCTION_IF(tstate->c_recursion_remaining <= 0, CALL, INLINE_CACHE_ENTRIES_CALL); + if (tstate->c_recursion_remaining <= 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } _PyStackRef arg_stackref = args[1]; _PyStackRef self_stackref = args[0]; - GO_TO_INSTRUCTION_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -3773,7 +4194,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_NON_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3787,8 +4209,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PA { callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - GO_TO_INSTRUCTION_IF(PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + if (PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _CALL_NON_PY_GENERAL { @@ -3891,16 +4321,28 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PAR /* Skip 1 cache entry */ // _CHECK_PEP_523 { - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - GO_TO_INSTRUCTION_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyFunctionObject *func = (PyFunctionObject *)callable_o; - GO_TO_INSTRUCTION_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _CHECK_FUNCTION_EXACT_ARGS { @@ -3909,15 +4351,27 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PAR assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - GO_TO_INSTRUCTION_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL, INLINE_CACHE_ENTRIES_CALL); + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(tstate->py_recursion_remaining <= 1, CALL, INLINE_CACHE_ENTRIES_CALL); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _INIT_CALL_PY_EXACT_ARGS { @@ -3991,16 +4445,28 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS /* Skip 1 cache entry */ // _CHECK_PEP_523 { - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, CALL, INLINE_CACHE_ENTRIES_CALL); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - GO_TO_INSTRUCTION_IF(!PyFunction_Check(callable_o), CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyFunctionObject *func = (PyFunctionObject *)callable_o; - GO_TO_INSTRUCTION_IF(func->func_version != func_version, CALL, INLINE_CACHE_ENTRIES_CALL); + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _PY_FRAME_GENERAL { @@ -4076,7 +4542,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_STR_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -4094,8 +4561,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS){ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (callable_o != (PyObject *)&PyUnicode_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Str(arg_o); @@ -4145,7 +4620,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TUPLE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -4163,8 +4639,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(callable_o != (PyObject *)&PyTuple_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (callable_o != (PyObject *)&PyTuple_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PySequence_Tuple(arg_o); @@ -4214,7 +4698,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TYPE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -4230,8 +4715,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS){ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - GO_TO_INSTRUCTION_IF(!PyStackRef_IsNull(null), CALL, INLINE_CACHE_ENTRIES_CALL); - GO_TO_INSTRUCTION_IF(callable_o != (PyObject *)&PyType_Type, CALL, INLINE_CACHE_ENTRIES_CALL); + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (callable_o != (PyObject *)&PyType_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); PyStackRef_CLOSE(arg); @@ -4510,7 +5003,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_FLOAT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); @@ -4523,8 +5017,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAM left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - GO_TO_INSTRUCTION_IF(!PyFloat_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!PyFloat_CheckExact(right_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } /* Skip 1 cache entry */ // _COMPARE_OP_FLOAT @@ -4566,7 +5068,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_INT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); @@ -4579,16 +5082,32 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS) left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + if (!PyLong_CheckExact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } /* Skip 1 cache entry */ // _COMPARE_OP_INT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - GO_TO_INSTRUCTION_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + if (!_PyLong_IsCompact((PyLongObject *)left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!_PyLong_IsCompact((PyLongObject *)right_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && _PyLong_DigitCount((PyLongObject *)right_o) <= 1); @@ -4626,7 +5145,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_STR); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); @@ -4639,8 +5159,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS) left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); - GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP, INLINE_CACHE_ENTRIES_COMPARE_OP); + if (!PyUnicode_CheckExact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!PyUnicode_CheckExact(right_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } /* Skip 1 cache entry */ // _COMPARE_OP_STR @@ -4747,7 +5275,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_DICT); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); @@ -4759,7 +5288,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAM left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(right_o), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); + if (!PyDict_CheckExact(right_o)) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + Py_MUSTTAIL return _TAIL_CALL_CONTAINS_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); @@ -4793,7 +5326,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_SET); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); @@ -4805,7 +5339,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - GO_TO_INSTRUCTION_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP); + if (!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o))) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + Py_MUSTTAIL return _TAIL_CALL_CONTAINS_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5680,7 +6218,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_GEN); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); @@ -5690,14 +6229,26 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS){ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _FOR_ITER_GEN_FRAME { iter = stack_pointer[-1]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - GO_TO_INSTRUCTION_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); - GO_TO_INSTRUCTION_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + if (Py_TYPE(gen) != &PyGen_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(FOR_ITER, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, PyStackRef_None); @@ -5746,7 +6297,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_LIST); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); @@ -5756,7 +6308,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS){ // _ITER_CHECK_LIST { iter = stack_pointer[-1]; - GO_TO_INSTRUCTION_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _ITER_JUMP_LIST { @@ -5815,7 +6371,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_RANGE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); @@ -5826,7 +6383,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS) { iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - GO_TO_INSTRUCTION_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _ITER_JUMP_RANGE { @@ -5876,7 +6437,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_TUPLE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); @@ -5886,7 +6448,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS) // _ITER_CHECK_TUPLE { iter = stack_pointer[-1]; - GO_TO_INSTRUCTION_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER, INLINE_CACHE_ENTRIES_FOR_ITER); + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _ITER_JUMP_TUPLE { @@ -7842,9 +8408,17 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - GO_TO_INSTRUCTION_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (!PyType_Check(owner_o)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS @@ -7899,16 +8473,28 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_C owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - GO_TO_INSTRUCTION_IF(!PyType_Check(owner_o), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (!PyType_Check(owner_o)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _GUARD_TYPE_VERSION { uint32_t type_version = read_u32(&this_instr[4].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _LOAD_ATTR_CLASS { @@ -7961,17 +8547,33 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDE PyObject *getattribute = read_obj(&this_instr[6].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert((oparg & 1) == 0); - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); - GO_TO_INSTRUCTION_IF(f->func_version != func_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (f->func_version != func_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 2); - GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( @@ -8018,14 +8620,22 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _CHECK_MANAGED_OBJECT_HAS_VALUES { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - GO_TO_INSTRUCTION_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _LOAD_ATTR_INSTANCE_VALUE { @@ -8033,10 +8643,18 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); - GO_TO_INSTRUCTION_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } #ifdef Py_GIL_DISABLED if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { - GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } #else attr = PyStackRef_FromPyObjectNew(attr_o); @@ -8090,7 +8708,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _CHECK_ATTR_METHOD_LAZY_DICT { @@ -8098,7 +8720,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_ char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); /* This object has a __dict__, just not yet created */ - GO_TO_INSTRUCTION_IF(dict != NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (dict != NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } /* Skip 1 cache entry */ // _LOAD_ATTR_METHOD_LAZY_DICT @@ -8152,7 +8778,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CA uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_NO_DICT @@ -8207,14 +8837,22 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAI uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyDictValues *ivs = _PyObject_InlineValues(owner_o); - GO_TO_INSTRUCTION_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _GUARD_KEYS_VERSION { @@ -8222,7 +8860,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAI PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _LOAD_ATTR_METHOD_WITH_VALUES { @@ -8276,11 +8918,19 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM owner = stack_pointer[-1]; uint32_t dict_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - GO_TO_INSTRUCTION_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } mod_keys = keys; } // _LOAD_ATTR_MODULE_FROM_KEYS @@ -8291,11 +8941,19 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); // Clear mod_keys from stack in case we need to deopt - GO_TO_INSTRUCTION_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); if (!increfed) { - GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } #else Py_INCREF(attr_o); @@ -8349,7 +9007,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT( uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT @@ -8399,14 +9061,22 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VAL uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyDictValues *ivs = _PyObject_InlineValues(owner_o); - GO_TO_INSTRUCTION_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _GUARD_KEYS_VERSION { @@ -8414,7 +9084,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VAL PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { @@ -8458,7 +9132,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR /* Skip 1 cache entry */ // _CHECK_PEP_523 { - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _GUARD_TYPE_VERSION { @@ -8466,7 +9144,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } /* Skip 2 cache entries */ // _LOAD_ATTR_PROPERTY_FRAME @@ -8476,10 +9158,26 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; PyCodeObject *code = (PyCodeObject *)f->func_code; - GO_TO_INSTRUCTION_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - GO_TO_INSTRUCTION_IF(code->co_kwonlyargcount, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - GO_TO_INSTRUCTION_IF(code->co_argcount != 1, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); - GO_TO_INSTRUCTION_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (code->co_kwonlyargcount) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (code->co_argcount != 1) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(LOAD_ATTR, hit); new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); new_frame->localsplus[0] = owner; @@ -8547,7 +9245,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _LOAD_ATTR_SLOT { @@ -8555,10 +9257,18 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **addr = (PyObject **)((char *)owner_o + index); PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); - GO_TO_INSTRUCTION_IF(attr_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); - GO_TO_INSTRUCTION_IF(!increfed, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } #else attr = PyStackRef_FromPyObjectNew(attr_o); #endif @@ -8612,14 +9322,22 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _CHECK_ATTR_WITH_HINT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); - GO_TO_INSTRUCTION_IF(dict_o == NULL, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (dict_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } assert(PyDict_CheckExact((PyObject *)dict_o)); dict = dict_o; } @@ -8628,26 +9346,46 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA uint16_t hint = read_u16(&this_instr[4].cache); PyObject *attr_o; if (!LOCK_OBJECT(dict)) { - GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } if (hint >= (size_t)dict->ma_keys->dk_nentries) { UNLOCK_OBJECT(dict); - GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); - GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } attr_o = ep->me_value; if (attr_o == NULL) { UNLOCK_OBJECT(dict); - GO_TO_INSTRUCTION_IF(true, LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); @@ -9279,18 +10017,34 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } assert(DK_IS_UNICODE(keys)); } // _GUARD_BUILTINS_VERSION_PUSH_KEYS { uint16_t version = read_u16(&this_instr[3].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); - GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } builtins_keys = keys; assert(DK_IS_UNICODE(builtins_keys)); } @@ -9299,10 +10053,18 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - GO_TO_INSTRUCTION_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + if (res_o == NULL) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - GO_TO_INSTRUCTION_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -9352,9 +10114,17 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PAR { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } globals_keys = keys; assert(DK_IS_UNICODE(globals_keys)); } @@ -9364,10 +10134,18 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PAR uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - GO_TO_INSTRUCTION_IF(res_o == NULL, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + if (res_o == NULL) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - GO_TO_INSTRUCTION_IF(!increfed, LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -9684,7 +10462,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); @@ -9700,8 +10479,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_P PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(!(oparg & 1)); - GO_TO_INSTRUCTION_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); - GO_TO_INSTRUCTION_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + if (global_super != (PyObject *)&PySuper_Type) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_SUPER_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!PyType_Check(class)) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_SUPER_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9737,7 +10524,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_P Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); @@ -9754,8 +10542,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(oparg & 1); - GO_TO_INSTRUCTION_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); - GO_TO_INSTRUCTION_IF(!PyType_Check(class), LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); + if (global_super != (PyObject *)&PySuper_Type) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_SUPER_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!PyType_Check(class)) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_LOAD_SUPER_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -10690,21 +11486,34 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(RESUME_CHECK); static_assert(0 == 0, "incorrect cache size"); #if defined(__EMSCRIPTEN__) - GO_TO_INSTRUCTION_IF(_Py_emscripten_signal_clock == 0, RESUME, 0); + if (_Py_emscripten_signal_clock == 0) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + Py_MUSTTAIL return _TAIL_CALL_RESUME(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); - GO_TO_INSTRUCTION_IF(eval_breaker != version, RESUME, 0); + if (eval_breaker != version) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + Py_MUSTTAIL return _TAIL_CALL_RESUME(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } #ifdef Py_GIL_DISABLED - GO_TO_INSTRUCTION_IF(frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME, 0); + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + Py_MUSTTAIL return _TAIL_CALL_RESUME(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } #endif } DISPATCH(); @@ -10933,7 +11742,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(SEND_GEN); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); @@ -10944,15 +11754,27 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS){ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - GO_TO_INSTRUCTION_IF(tstate->interp->eval_frame, SEND, INLINE_CACHE_ENTRIES_SEND); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + Py_MUSTTAIL return _TAIL_CALL_SEND(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _SEND_GEN_FRAME { v = stack_pointer[-1]; receiver = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - GO_TO_INSTRUCTION_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND, INLINE_CACHE_ENTRIES_SEND); - GO_TO_INSTRUCTION_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND, INLINE_CACHE_ENTRIES_SEND); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + Py_MUSTTAIL return _TAIL_CALL_SEND(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + Py_MUSTTAIL return _TAIL_CALL_SEND(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(SEND, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, v); @@ -11251,11 +12073,19 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_C uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + if (!LOCK_OBJECT(owner_o)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } PyTypeObject *tp = Py_TYPE(owner_o); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UNLOCK_OBJECT(owner_o); - GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } } // _GUARD_DORV_NO_DICT @@ -11266,7 +12096,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_C if (_PyObject_GetManagedDict(owner_o) || !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UNLOCK_OBJECT(owner_o); - GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } } // _STORE_ATTR_INSTANCE_VALUE @@ -11328,14 +12162,22 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _STORE_ATTR_SLOT { value = stack_pointer[-2]; uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(owner_o), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + if (!LOCK_OBJECT(owner_o)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; @@ -11384,7 +12226,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _STORE_ATTR_WITH_HINT { @@ -11393,12 +12239,24 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - GO_TO_INSTRUCTION_IF(dict == NULL, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); - GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(dict), STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + if (dict == NULL) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!LOCK_OBJECT(dict)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } #ifdef Py_GIL_DISABLED if (dict != _PyObject_GetManagedDict(owner_o)) { UNLOCK_OBJECT(dict); - GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } #endif assert(PyDict_CheckExact((PyObject *)dict)); @@ -11406,17 +12264,29 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P if (hint >= (size_t)dict->ma_keys->dk_nentries || !DK_IS_UNICODE(dict->ma_keys)) { UNLOCK_OBJECT(dict); - GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } PyObject *old_value = ep->me_value; if (old_value == NULL) { UNLOCK_OBJECT(dict); - GO_TO_INSTRUCTION_IF(true, STORE_ATTR, INLINE_CACHE_ENTRIES_STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } _PyFrame_SetStackPointer(frame, stack_pointer); _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); @@ -11802,7 +12672,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); @@ -11814,7 +12685,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARA dict_st = stack_pointer[-2]; value = stack_pointer[-3]; PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - GO_TO_INSTRUCTION_IF(!PyDict_CheckExact(dict), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(STORE_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, @@ -11847,7 +12722,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); @@ -11860,16 +12736,36 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_ value = stack_pointer[-3]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); - GO_TO_INSTRUCTION_IF(!PyList_CheckExact(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + if (!PyLong_CheckExact(sub)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!PyList_CheckExact(list)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } // Ensure nonnegative, zero-or-one-digit ints. - GO_TO_INSTRUCTION_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(list), STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + if (!LOCK_OBJECT(list)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } // Ensure index < len(list) if (index >= PyList_GET_SIZE(list)) { UNLOCK_OBJECT(list); - GO_TO_INSTRUCTION_IF(true, STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR); + if (true) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); @@ -12014,7 +12910,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PA uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - GO_TO_INSTRUCTION_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } // _REPLACE_WITH_TRUE { @@ -12045,7 +12945,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_BOOL); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); @@ -12053,7 +12954,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS){ /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; - GO_TO_INSTRUCTION_IF(!PyStackRef_BoolCheck(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + if (!PyStackRef_BoolCheck(value)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(TO_BOOL, hit); } DISPATCH(); @@ -12077,7 +12982,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_INT); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); @@ -12087,7 +12993,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS){ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - GO_TO_INSTRUCTION_IF(!PyLong_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + if (!PyLong_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value_o)) { assert(_Py_IsImmortal(value_o)); @@ -12120,7 +13030,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_LIST); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); @@ -12130,7 +13041,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS){ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - GO_TO_INSTRUCTION_IF(!PyList_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + if (!PyList_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(TO_BOOL, hit); res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; PyStackRef_CLOSE(value); @@ -12157,7 +13072,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_NONE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); @@ -12167,7 +13083,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS){ /* Skip 2 cache entries */ value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - GO_TO_INSTRUCTION_IF(!PyStackRef_IsNone(value), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + if (!PyStackRef_IsNone(value)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(TO_BOOL, hit); res = PyStackRef_False; stack_pointer[-1] = res; @@ -12193,7 +13113,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS){ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_STR); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); @@ -12203,7 +13124,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS){ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - GO_TO_INSTRUCTION_IF(!PyUnicode_CheckExact(value_o), TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(TO_BOOL, hit); if (value_o == &_Py_STR(empty)) { assert(_Py_IsImmortal(value_o)); @@ -12437,7 +13362,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); @@ -12447,11 +13373,23 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_P seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - GO_TO_INSTRUCTION_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); - GO_TO_INSTRUCTION_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + if (!PyList_CheckExact(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (!LOCK_OBJECT(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } if (PyList_GET_SIZE(seq_o) != oparg) { UNLOCK_OBJECT(seq_o); - GO_TO_INSTRUCTION_IF(true, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + if (true) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq_o); @@ -12484,7 +13422,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_P Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); @@ -12494,8 +13433,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - GO_TO_INSTRUCTION_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); - GO_TO_INSTRUCTION_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + if (!PyTuple_CheckExact(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (PyTuple_GET_SIZE(seq_o) != oparg) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { @@ -12526,7 +13473,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); @@ -12537,8 +13485,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_C seq = stack_pointer[-1]; assert(oparg == 2); PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - GO_TO_INSTRUCTION_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); - GO_TO_INSTRUCTION_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE, INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + if (!PyTuple_CheckExact(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } + if (PyTuple_GET_SIZE(seq_o) != 2) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, opcode, oparg); + } STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 9907c80de95d39..673e3b1a4dc972 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -150,20 +150,21 @@ def deopt_if( storage: Storage, inst: Instruction | None, ) -> bool: - self.out.emit_at("GO_TO_INSTRUCTION_IF", tkn) + self.out.start_line() + self.out.emit("if (") lparen = next(tkn_iter) - self.emit(lparen) assert lparen.kind == "LPAREN" first_tkn = tkn_iter.peek() emit_to(self.out, tkn_iter, "RPAREN") + self.emit(") {\n") next(tkn_iter) # Semi colon - self.out.emit(", ") assert inst is not None assert inst.family is not None - self.out.emit(inst.family.name) - self.out.emit(", ") - self.out.emit(inst.family.size) - self.out.emit(");\n") + family_name = inst.family.name + self.emit(f"UPDATE_MISS_STATS({family_name});\n") + self.emit(f"assert(_PyOpcode_Deopt[opcode] == ({family_name}));\n") + self.emit(f"goto PREDICTED_{family_name};\n") + self.emit("}\n") return not always_true(first_tkn) exit_if = deopt_if diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 791cf73b529e44..0240ffba3a9ac2 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -129,6 +129,9 @@ def uses_this(inst: Instruction) -> bool: for cache in uop.caches: if cache.name != "unused": return True + for tkn in uop.body: + if tkn.kind == "IDENTIFIER" and (tkn.text == "DEOPT_IF" or tkn.text == "EXIT_IF"): + return True return False diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index 2c65b2a062fa0b..931d2fe297a8fe 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -14,6 +14,8 @@ CWriter, Emitter, TokenIterator, + emit_to, + always_true, ) from analyzer import ( @@ -67,6 +69,34 @@ def go_to_instruction( self.emit(f"Py_MUSTTAIL return (INSTRUCTION_TABLE[{name.text}])(frame, stack_pointer, tstate, next_instr - 1 - {size}, opcode, oparg);\n") return True + def deopt_if( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + self.out.start_line() + self.out.emit("if (") + lparen = next(tkn_iter) + assert lparen.kind == "LPAREN" + first_tkn = tkn_iter.peek() + emit_to(self.out, tkn_iter, "RPAREN") + self.emit(") {\n") + next(tkn_iter) # Semi colon + assert inst is not None + assert inst.family is not None + family_name = inst.family.name + self.emit(f"UPDATE_MISS_STATS({family_name});\n") + self.emit(f"assert(_PyOpcode_Deopt[opcode] == ({family_name}));\n") + self.emit(f"Py_MUSTTAIL return _TAIL_CALL_{family_name}(frame, stack_pointer, tstate, this_instr, opcode, oparg);\n") + self.emit("}\n") + return not always_true(first_tkn) + + + exit_if = deopt_if + class TailCallLabelsEmitter(Emitter): def __init__(self, out: CWriter): super().__init__(out) From 21dfba03ad4436ea22d02d40d36e5ba71cd63586 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 30 Jan 2025 20:18:42 +0800 Subject: [PATCH 084/110] Fix tests --- Lib/test/test_generated_cases.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index fad5ef89cb3d1e..cbc1016ee44569 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -459,6 +459,11 @@ def test_predictions(self): INSTRUCTION_STATS(OP3); static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size"); _PyStackRef res; + if (xxx) { + UPDATE_MISS_STATS(OP1); + assert(_PyOpcode_Deopt[opcode] == (OP1)); + goto PREDICTED_OP1; + } GO_TO_INSTRUCTION_IF(xxx, OP1, INLINE_CACHE_ENTRIES_OP1); res = Py_None; stack_pointer[-1] = res; From 158a4c46c6cb674e1eb56a731cfbefa8a85bb2fd Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 30 Jan 2025 20:25:43 +0800 Subject: [PATCH 085/110] Cleanup --- Lib/test/test_generated_cases.py | 1 - Python/generated_cases.c.h | 156 ++++++------------ Tools/cases_generator/tier1_generator.py | 9 +- .../tier1_tail_call_generator.py | 17 +- 4 files changed, 70 insertions(+), 113 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index cbc1016ee44569..5970972d855e75 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -464,7 +464,6 @@ def test_predictions(self): assert(_PyOpcode_Deopt[opcode] == (OP1)); goto PREDICTED_OP1; } - GO_TO_INSTRUCTION_IF(xxx, OP1, INLINE_CACHE_ENTRIES_OP1); res = Py_None; stack_pointer[-1] = res; DISPATCH(); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6505d48f15d4c2..f14dbd72d583a2 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -69,8 +69,7 @@ } TARGET(BINARY_OP_ADD_FLOAT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -116,8 +115,7 @@ } TARGET(BINARY_OP_ADD_INT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -162,8 +160,7 @@ } TARGET(BINARY_OP_ADD_UNICODE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -259,8 +256,7 @@ } TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -335,8 +331,7 @@ } TARGET(BINARY_OP_MULTIPLY_FLOAT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -382,8 +377,7 @@ } TARGET(BINARY_OP_MULTIPLY_INT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -428,8 +422,7 @@ } TARGET(BINARY_OP_SUBTRACT_FLOAT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -475,8 +468,7 @@ } TARGET(BINARY_OP_SUBTRACT_INT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -618,8 +610,7 @@ } TARGET(BINARY_SUBSCR_DICT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); @@ -658,8 +649,7 @@ } TARGET(BINARY_SUBSCR_GETITEM) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); @@ -738,8 +728,7 @@ } TARGET(BINARY_SUBSCR_LIST_INT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); @@ -799,8 +788,7 @@ } TARGET(BINARY_SUBSCR_STR_INT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); @@ -852,8 +840,7 @@ } TARGET(BINARY_SUBSCR_TUPLE_INT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); @@ -1629,8 +1616,7 @@ } TARGET(CALL_BUILTIN_CLASS) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_CLASS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -1716,8 +1702,7 @@ } TARGET(CALL_BUILTIN_FAST) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -1809,8 +1794,7 @@ } TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -1903,8 +1887,7 @@ } TARGET(CALL_BUILTIN_O) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -2194,8 +2177,7 @@ } TARGET(CALL_ISINSTANCE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ISINSTANCE); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -2535,8 +2517,7 @@ } TARGET(CALL_KW_NON_PY) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_NON_PY); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); @@ -2736,8 +2717,7 @@ } TARGET(CALL_LEN) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LEN); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -2792,8 +2772,7 @@ } TARGET(CALL_LIST_APPEND) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LIST_APPEND); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -2843,8 +2822,7 @@ } TARGET(CALL_METHOD_DESCRIPTOR_FAST) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -2942,8 +2920,7 @@ } TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3041,8 +3018,7 @@ } TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3134,8 +3110,7 @@ } TARGET(CALL_METHOD_DESCRIPTOR_O) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3230,8 +3205,7 @@ } TARGET(CALL_NON_PY_GENERAL) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_NON_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3524,8 +3498,7 @@ } TARGET(CALL_STR_1) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_STR_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3584,8 +3557,7 @@ } TARGET(CALL_TUPLE_1) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TUPLE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3644,8 +3616,7 @@ } TARGET(CALL_TYPE_1) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TYPE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3859,8 +3830,7 @@ } TARGET(COMPARE_OP_FLOAT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_FLOAT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); @@ -3906,8 +3876,7 @@ } TARGET(COMPARE_OP_INT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_INT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); @@ -3965,8 +3934,7 @@ } TARGET(COMPARE_OP_STR) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_STR); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); @@ -4059,8 +4027,7 @@ } TARGET(CONTAINS_OP_DICT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_DICT); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); @@ -4092,8 +4059,7 @@ } TARGET(CONTAINS_OP_SET) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_SET); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); @@ -4606,8 +4572,7 @@ } TARGET(FOR_ITER_GEN) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_GEN); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); @@ -4667,8 +4632,7 @@ } TARGET(FOR_ITER_LIST) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_LIST); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); @@ -4723,8 +4687,7 @@ } TARGET(FOR_ITER_RANGE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_RANGE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); @@ -4771,8 +4734,7 @@ } TARGET(FOR_ITER_TUPLE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_TUPLE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); @@ -7500,8 +7462,7 @@ } TARGET(LOAD_SUPER_ATTR_ATTR) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); @@ -7544,8 +7505,7 @@ } TARGET(LOAD_SUPER_ATTR_METHOD) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); @@ -8092,8 +8052,7 @@ } TARGET(RESUME_CHECK) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME_CHECK); static_assert(0 == 0, "incorrect cache size"); @@ -8276,8 +8235,7 @@ } TARGET(SEND_GEN) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND_GEN); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); @@ -8900,8 +8858,7 @@ } TARGET(STORE_SUBSCR_DICT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); @@ -8932,8 +8889,7 @@ } TARGET(STORE_SUBSCR_LIST_INT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); @@ -9083,8 +9039,7 @@ } TARGET(TO_BOOL_BOOL) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_BOOL); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); @@ -9102,8 +9057,7 @@ } TARGET(TO_BOOL_INT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_INT); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); @@ -9132,8 +9086,7 @@ } TARGET(TO_BOOL_LIST) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_LIST); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); @@ -9156,8 +9109,7 @@ } TARGET(TO_BOOL_NONE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_NONE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); @@ -9179,8 +9131,7 @@ } TARGET(TO_BOOL_STR) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_STR); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); @@ -9320,8 +9271,7 @@ } TARGET(UNPACK_SEQUENCE_LIST) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); @@ -9362,8 +9312,7 @@ } TARGET(UNPACK_SEQUENCE_TUPLE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); @@ -9395,8 +9344,7 @@ } TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 0240ffba3a9ac2..d3c8f79a388efd 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -24,7 +24,7 @@ Emitter, ) from cwriter import CWriter -from typing import TextIO +from typing import TextIO, Callable from stack import Local, Stack, StackError, get_stack_effect, Storage @@ -129,13 +129,10 @@ def uses_this(inst: Instruction) -> bool: for cache in uop.caches: if cache.name != "unused": return True - for tkn in uop.body: - if tkn.kind == "IDENTIFIER" and (tkn.text == "DEOPT_IF" or tkn.text == "EXIT_IF"): - return True return False -def write_single_inst(out: CWriter, emitter: Emitter, name: str, inst: Instruction) -> None: +def write_single_inst(out: CWriter, emitter: Emitter, name: str, inst: Instruction, uses_this: Callable[Instruction, bool]) -> None: needs_this = uses_this(inst) unused_guard = "(void)this_instr;\n" if inst.properties.needs_prev: @@ -243,7 +240,7 @@ def generate_tier1_cases( for name, inst in sorted(analysis.instructions.items()): out.emit("\n") out.emit(f"TARGET({name}) {{\n") - write_single_inst(out, emitter, name, inst) + write_single_inst(out, emitter, name, inst, uses_this) if not inst.parts[-1].properties.always_exits: out.emit("DISPATCH();\n") out.start_line() diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index 931d2fe297a8fe..4dd61b12385b6f 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -4,7 +4,6 @@ """ import argparse -import re from typing import TextIO @@ -144,6 +143,20 @@ def generate_label_handlers( emitter.emit("\n") +def uses_this(inst: Instruction) -> bool: + if inst.properties.needs_this: + return True + for uop in inst.parts: + if not isinstance(uop, Uop): + continue + for cache in uop.caches: + if cache.name != "unused": + return True + for tkn in uop.body: + if tkn.kind == "IDENTIFIER" and (tkn.text == "DEOPT_IF" or tkn.text == "EXIT_IF"): + return True + return False + def generate_tier1( filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool ) -> None: @@ -175,7 +188,7 @@ def generate_tier1( # escaping locals. # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=118430#c1 out.emit("{\n") - write_single_inst(out, emitter, name, inst) + write_single_inst(out, emitter, name, inst, uses_this) out.emit("}\n") if not inst.parts[-1].properties.always_exits: out.emit("DISPATCH();\n") From 1463132284fd9bf38a119bafa7c8ddedb1dc62fc Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 31 Jan 2025 19:55:30 +0800 Subject: [PATCH 086/110] Apply review suggestions on type checking --- Python/generated_tail_call_handlers.c.h | 10 ++-- Tools/cases_generator/analyzer.py | 1 + Tools/cases_generator/generators_common.py | 46 ++++++++++--------- Tools/cases_generator/tier1_generator.py | 8 +++- .../tier1_tail_call_generator.py | 24 ++++------ Tools/cases_generator/tier2_generator.py | 11 +++-- 6 files changed, 52 insertions(+), 48 deletions(-) diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index f616a59aee313b..6898da36bdb879 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -7049,11 +7049,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS){ { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - - Py_MUSTTAIL return (INSTRUCTION_TABLE[CALL_FUNCTION_EX])(frame, stack_pointer, tstate, next_instr - 1 - 0, opcode, oparg); + Py_MUSTTAIL return (INSTRUCTION_TABLE[CALL_FUNCTION_EX])(frame, stack_pointer, tstate, this_instr, opcode, oparg); } pop_4_error: TAIL_CALL(pop_4_error); @@ -7095,7 +7095,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_P stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - Py_MUSTTAIL return (INSTRUCTION_TABLE[CALL_KW])(frame, stack_pointer, tstate, next_instr - 1 - INLINE_CACHE_ENTRIES_CALL_KW, opcode, oparg); + Py_MUSTTAIL return (INSTRUCTION_TABLE[CALL_KW])(frame, stack_pointer, tstate, this_instr, opcode, oparg); } pop_4_error: TAIL_CALL(pop_4_error); @@ -7430,7 +7430,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAI // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - Py_MUSTTAIL return (INSTRUCTION_TABLE[LOAD_SUPER_ATTR])(frame, stack_pointer, tstate, next_instr - 1 - INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR, opcode, oparg); + Py_MUSTTAIL return (INSTRUCTION_TABLE[LOAD_SUPER_ATTR])(frame, stack_pointer, tstate, this_instr, opcode, oparg); } pop_4_error: TAIL_CALL(pop_4_error); diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index b9293ff4b19951..ed9c839e1edf06 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -262,6 +262,7 @@ def is_super(self) -> bool: class Label: name: str body: list[lexer.Token] + instruction_size: int | None = None @dataclass diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 673e3b1a4dc972..ef6a8eafea26b8 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -4,6 +4,7 @@ from analyzer import ( Instruction, Uop, + Label, Properties, StackItem, analysis_error, @@ -11,7 +12,7 @@ from cwriter import CWriter from typing import Callable, TextIO, Iterator, Iterable from lexer import Token -from stack import Storage, StackError +from stack import Storage, StackError, Stack # Set this to true for voluminous output showing state of stack and locals PRINT_STACKS = False @@ -90,7 +91,7 @@ def emit_to(out: CWriter, tkn_iter: TokenIterator, end: str) -> Token: ReplacementFunctionType = Callable[ - [Token, TokenIterator, Uop, Storage, Instruction | None], bool + [Token, TokenIterator, Uop | Label, Storage, Instruction | None], bool ] def always_true(tkn: Token | None) -> bool: @@ -135,7 +136,7 @@ def dispatch( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: @@ -146,7 +147,7 @@ def deopt_if( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: @@ -173,7 +174,7 @@ def error_if( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: @@ -222,7 +223,7 @@ def error_no_pop( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: @@ -236,7 +237,7 @@ def decref_inputs( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: @@ -272,7 +273,7 @@ def kill_inputs( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: @@ -287,7 +288,7 @@ def kill( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: @@ -327,7 +328,7 @@ def stackref_close( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: @@ -347,7 +348,7 @@ def stackref_close_specialized( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: @@ -377,7 +378,7 @@ def stackref_steal( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: @@ -397,7 +398,7 @@ def sync_sp( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: @@ -413,7 +414,7 @@ def go_to_instruction( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: @@ -434,7 +435,7 @@ def save_stack( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: @@ -448,7 +449,7 @@ def pop_input( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: @@ -475,7 +476,7 @@ def reload_stack( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: @@ -488,7 +489,7 @@ def reload_stack( def instruction_size(self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: @@ -655,15 +656,16 @@ def emit_tokens( raise analysis_error(ex.args[0], rbrace) from None return storage - def emit_tokens_simple( + def emit_label( self, - tokens: list[Token] + label: Label ) -> None: - tkn_iter = TokenIterator(tokens) + tkn_iter = TokenIterator(label.body) self.out.start_line() for tkn in tkn_iter: if tkn.text in self._replacers: - self._replacers[tkn.text](tkn, tkn_iter, None, None, None) # type: ignore[arg-type] + storage = Storage(Stack(), [],[], []) + self._replacers[tkn.text](tkn, tkn_iter, label, storage, None) continue self.out.emit(tkn) diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index d3c8f79a388efd..fcea7deff85f7f 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -132,7 +132,13 @@ def uses_this(inst: Instruction) -> bool: return False -def write_single_inst(out: CWriter, emitter: Emitter, name: str, inst: Instruction, uses_this: Callable[Instruction, bool]) -> None: +def write_single_inst( + out: CWriter, + emitter: Emitter, + name: str, + inst: Instruction, + uses_this: Callable[[Instruction], bool] +) -> None: needs_this = uses_this(inst) unused_guard = "(void)this_instr;\n" if inst.properties.needs_prev: diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index 4dd61b12385b6f..1a56df6b1bfa50 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -22,6 +22,7 @@ Instruction, analyze_files, Uop, + Label, ) from tier1_generator import ( @@ -47,7 +48,7 @@ def go_to_instruction( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: @@ -56,23 +57,15 @@ def go_to_instruction( next(tkn_iter) next(tkn_iter) assert name.kind == "IDENTIFIER" - self.emit("\n") - inst = self.analysis.instructions[name.text] - fam = None - # Search for the family (if any) - for family_name, family in self.analysis.families.items(): - if inst.name == family_name: - fam = family - break - size = fam.size if fam is not None else 0 - self.emit(f"Py_MUSTTAIL return (INSTRUCTION_TABLE[{name.text}])(frame, stack_pointer, tstate, next_instr - 1 - {size}, opcode, oparg);\n") + self.out.start_line() + self.emit(f"Py_MUSTTAIL return (INSTRUCTION_TABLE[{name.text}])(frame, stack_pointer, tstate, this_instr, opcode, oparg);\n") return True def deopt_if( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: @@ -107,7 +100,7 @@ def goto( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: @@ -137,7 +130,7 @@ def generate_label_handlers( emitter.emit("\n") for name, label in analysis.labels.items(): emitter.emit(f"{function_proto(name)}\n") - emitter.emit_tokens_simple(label.body) + emitter.emit_label(label) emitter.emit("\n") emitter.emit("\n") @@ -153,7 +146,8 @@ def uses_this(inst: Instruction) -> bool: if cache.name != "unused": return True for tkn in uop.body: - if tkn.kind == "IDENTIFIER" and (tkn.text == "DEOPT_IF" or tkn.text == "EXIT_IF"): + if (tkn.kind == "IDENTIFIER" + and (tkn.text in {"DEOPT_IF", "EXIT_IF", "GO_TO_INSTRUCTION"})): return True return False diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index dd16a1a7eb28b5..506a777537225e 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -12,6 +12,7 @@ analyze_files, StackItem, analysis_error, + Label, ) from generators_common import ( DEFAULT_INPUT, @@ -73,7 +74,7 @@ def error_if( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: @@ -95,7 +96,7 @@ def error_no_pop( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: @@ -109,7 +110,7 @@ def deopt_if( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: @@ -130,7 +131,7 @@ def exit_if( # type: ignore[override] self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: @@ -150,7 +151,7 @@ def oparg( self, tkn: Token, tkn_iter: TokenIterator, - uop: Uop, + uop: Uop | Label, storage: Storage, inst: Instruction | None, ) -> bool: From f906c97c588a15b4a7773b7dc37f38019791f2c6 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 31 Jan 2025 20:22:15 +0800 Subject: [PATCH 087/110] Add start_frame label --- Python/bytecodes.c | 22 ++ Python/ceval.c | 2 - Python/ceval_macros.h | 19 - Python/generated_cases.c.h | 21 ++ Python/generated_tail_call_handlers.c.h | 466 ++++++++++++++++++++++++ 5 files changed, 509 insertions(+), 21 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 371caf89c127be..5b7fdeb6b7fc75 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5194,6 +5194,28 @@ dummy_func( assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); } + label(start_frame) { + if (_Py_EnterRecursivePy(tstate)) { + goto exit_unwind; + } + + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + +#ifdef Py_DEBUG + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + frame->lltrace = lltrace; + if (lltrace < 0) { + goto exit_unwind; + } +#endif +#if defined(Py_TAIL_CALL_INTERP) && !defined(IN_TAIL_CALL_INTERP) + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0); +#else + DISPATCH(); +#endif + } + label(pop_4_error) { STACK_SHRINK(1); goto pop_3_error; diff --git a/Python/ceval.c b/Python/ceval.c index 963726cf694590..027870ffe64009 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -866,13 +866,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int const _PyUOpInstruction *next_uop = NULL; #endif -start_frame: if (_Py_EnterRecursivePy(tstate)) { goto exit_unwind; } next_instr = frame->instr_ptr; -resume_frame: stack_pointer = _PyFrame_GetStackPointer(frame); LLTRACE_RESUME_FRAME(); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 1ffe31774e1951..5562a1dc157fa4 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -140,24 +140,6 @@ do { \ DISPATCH_GOTO(); \ } -#ifdef Py_TAIL_CALL_INTERP -#define DISPATCH_INLINED(NEW_FRAME) \ - do { \ - assert(tstate->interp->eval_frame == NULL); \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - assert((NEW_FRAME)->previous == frame); \ - frame = tstate->current_frame = (NEW_FRAME); \ - CALL_STAT_INC(inlined_py_calls); \ - if (_Py_EnterRecursivePy(tstate)) {\ - goto exit_unwind;\ - } \ - next_instr = frame->instr_ptr; \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - LLTRACE_RESUME_FRAME(); \ - NEXTOPARG(); \ - DISPATCH_GOTO(); \ - } while (0) -#else #define DISPATCH_INLINED(NEW_FRAME) \ do { \ assert(tstate->interp->eval_frame == NULL); \ @@ -167,7 +149,6 @@ do { \ CALL_STAT_INC(inlined_py_calls); \ goto start_frame; \ } while (0) -#endif // Use this instead of 'goto error' so Tier 2 can go to a different label #define GOTO_ERROR(LABEL) goto LABEL diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f14dbd72d583a2..cadebc31754f5e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -9498,6 +9498,27 @@ #endif /* Py_TAIL_CALL_INTERP */ /* BEGIN LABELS */ + start_frame: + { + if (_Py_EnterRecursivePy(tstate)) { + goto exit_unwind; + } + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + #ifdef Py_DEBUG + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + frame->lltrace = lltrace; + if (lltrace < 0) { + goto exit_unwind; + } + #endif + #if defined(Py_TAIL_CALL_INTERP) && !defined(IN_TAIL_CALL_INTERP) + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0); + #else + DISPATCH(); + #endif + } + pop_4_error: { STACK_SHRINK(1); diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 6898da36bdb879..9557ca75c37bb4 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -11,6 +11,7 @@ static inline PyObject *_TAIL_CALL_shim(TAIL_CALL_PARAMS); static py_tail_call_funcptr INSTRUCTION_TABLE[256]; +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_4_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_3_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); @@ -20,6 +21,27 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_resume_with_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS) +{ + if (_Py_EnterRecursivePy(tstate)) { + TAIL_CALL(exit_unwind); + } + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + #ifdef Py_DEBUG + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + frame->lltrace = lltrace; + if (lltrace < 0) { + TAIL_CALL(exit_unwind); + } + #endif + #if defined(Py_TAIL_CALL_INTERP) && !defined(IN_TAIL_CALL_INTERP) + return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0); + #else + DISPATCH(); + #endif +} + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_4_error(TAIL_CALL_PARAMS) { STACK_SHRINK(1); @@ -203,6 +225,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -268,6 +292,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -332,6 +358,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -396,6 +424,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -465,6 +495,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -559,6 +591,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -624,6 +658,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -688,6 +724,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -753,6 +791,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -817,6 +857,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -885,6 +927,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -950,6 +994,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1008,6 +1054,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PAR assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1106,6 +1154,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_ } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1185,6 +1235,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1256,6 +1308,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1323,6 +1377,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CAL assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1359,6 +1415,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1417,6 +1475,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1481,6 +1541,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1525,6 +1587,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1578,6 +1642,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1614,6 +1680,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1641,6 +1709,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS){ Py_FatalError("Executing a cache."); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1820,6 +1890,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1957,6 +2029,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_C } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -2111,6 +2185,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -2249,6 +2325,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_C } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -2354,6 +2432,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -2465,6 +2545,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -2577,6 +2659,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS( assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -2680,6 +2764,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS) assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -2861,6 +2947,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -2897,6 +2985,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAM stack_pointer[-1] = res; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -2940,6 +3030,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -3014,6 +3106,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -3190,6 +3284,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -3335,6 +3431,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_P } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -3456,6 +3554,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -3572,6 +3672,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -3646,6 +3748,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -3715,6 +3819,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -3832,6 +3938,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -3949,6 +4057,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -4060,6 +4170,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -4174,6 +4286,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -4289,6 +4403,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -4413,6 +4529,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PAR } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -4522,6 +4640,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -4600,6 +4720,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -4678,6 +4800,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -4733,6 +4857,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -4798,6 +4924,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS) stack_pointer[-1] = match; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -4844,6 +4972,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS stack_pointer[-1] = b; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -4904,6 +5034,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -4983,6 +5115,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5048,6 +5182,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5125,6 +5261,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS) assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5191,6 +5329,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS) assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5255,6 +5395,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5306,6 +5448,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5358,6 +5502,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5396,6 +5542,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS){ stack_pointer[-1] = result; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5429,6 +5577,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5465,6 +5615,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS) } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5500,6 +5652,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5538,6 +5692,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS){ stack_pointer = _PyFrame_GetStackPointer(frame); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5574,6 +5730,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS){ SETLOCAL(oparg, PyStackRef_NULL); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5614,6 +5772,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS){ } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5661,6 +5821,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS){ } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5700,6 +5862,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5747,6 +5911,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5798,6 +5964,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5847,6 +6015,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5881,6 +6051,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5917,6 +6089,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5968,6 +6142,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS) #endif /* _Py_TIER2 */ } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6006,6 +6182,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6035,6 +6213,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS){ PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); } + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6078,6 +6258,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS){ stack_pointer[-1] = res; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6118,6 +6300,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6198,6 +6382,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6277,6 +6463,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS){ } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6351,6 +6539,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6417,6 +6607,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS) assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6488,6 +6680,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS) assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6553,6 +6747,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS){ stack_pointer[-1] = iter; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6591,6 +6787,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6626,6 +6824,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS){ stack_pointer[-1] = iter; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6662,6 +6862,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS){ stack_pointer[-1] = iter; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6701,6 +6903,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6762,6 +6966,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PA stack_pointer[-1] = iter; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6799,6 +7005,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6842,6 +7050,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7029,6 +7239,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7055,6 +7267,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TA INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); Py_MUSTTAIL return (INSTRUCTION_TABLE[CALL_FUNCTION_EX])(frame, stack_pointer, tstate, this_instr, opcode, oparg); } + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7097,6 +7311,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_P PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); Py_MUSTTAIL return (INSTRUCTION_TABLE[CALL_KW])(frame, stack_pointer, tstate, this_instr, opcode, oparg); } + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7140,6 +7356,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_P assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7185,6 +7403,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7240,6 +7460,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_ } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7277,6 +7499,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CA opcode = next_opcode; DISPATCH_GOTO(); } + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7319,6 +7543,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_ } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7346,6 +7572,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_C INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7402,6 +7630,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARA opcode = original_opcode; DISPATCH_GOTO(); } + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7432,6 +7662,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAI PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); Py_MUSTTAIL return (INSTRUCTION_TABLE[LOAD_SUPER_ATTR])(frame, stack_pointer, tstate, this_instr, opcode, oparg); } + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7461,6 +7693,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7494,6 +7728,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7528,6 +7764,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(T } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7564,6 +7802,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TA } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7598,6 +7838,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NON } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7632,6 +7874,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TA } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7722,6 +7966,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PA } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7783,6 +8029,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_C assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7868,6 +8116,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7905,6 +8155,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAM /* Not strictly necessary, but prevents warnings */ return result; } + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7942,6 +8194,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8003,6 +8257,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS){ } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8085,6 +8341,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARA } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8117,6 +8375,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_ JUMPBY(-oparg); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8165,6 +8425,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_P } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8191,6 +8453,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS){ JUMPBY(oparg); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8225,6 +8489,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8280,6 +8546,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8374,6 +8642,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8439,6 +8709,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8514,6 +8786,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_C assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8585,6 +8859,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDE frame->return_offset = 10 ; DISPATCH_INLINED(new_frame); } + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8673,6 +8949,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8743,6 +9021,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8802,6 +9082,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8883,6 +9165,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAI assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8973,6 +9257,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9027,6 +9313,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT( stack_pointer[-1] = attr; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9102,6 +9390,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VAL stack_pointer[-1] = attr; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9210,6 +9500,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9286,6 +9578,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9403,6 +9697,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9445,6 +9741,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9485,6 +9783,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_P assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9536,6 +9836,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9569,6 +9871,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_PA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9601,6 +9905,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_MORTAL(TAIL_CALL_PARA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9639,6 +9945,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9670,6 +9978,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9702,6 +10012,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9742,6 +10054,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9777,6 +10091,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9830,6 +10146,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CAL stack_pointer[-1] = value; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9918,6 +10236,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_C stack_pointer[-1] = v; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9984,6 +10304,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10081,6 +10403,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10162,6 +10486,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PAR assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10200,6 +10526,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10235,6 +10563,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10267,6 +10597,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS) assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10322,6 +10654,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10442,6 +10776,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10504,6 +10840,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_P assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10578,6 +10916,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10611,6 +10951,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS){ SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell)); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10650,6 +10992,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS){ stack_pointer[-1] = func; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10695,6 +11039,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10751,6 +11097,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10791,6 +11139,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10824,6 +11174,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10857,6 +11209,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS) assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10882,6 +11236,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS){ INSTRUCTION_STATS(NOP); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10907,6 +11263,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS){ INSTRUCTION_STATS(NOT_TAKEN); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10942,6 +11300,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10972,6 +11332,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11007,6 +11369,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11058,6 +11422,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11109,6 +11475,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_P assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11144,6 +11512,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11174,6 +11544,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11217,6 +11589,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11247,6 +11621,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11290,6 +11666,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS){ } goto error; } + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11351,6 +11729,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS){ stack_pointer = _PyFrame_GetStackPointer(frame); goto exception_unwind; } + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11378,6 +11758,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS){ Py_FatalError("Executing RESERVED instruction."); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11466,6 +11848,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS){ } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11517,6 +11901,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS){ #endif } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11568,6 +11954,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11614,6 +12002,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11722,6 +12112,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS){ stack_pointer[-1] = retval; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11805,6 +12197,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS){ } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11860,6 +12254,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARA } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11897,6 +12293,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11939,6 +12337,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11976,6 +12376,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12039,6 +12441,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12128,6 +12532,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_C } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12192,6 +12598,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12305,6 +12713,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P } } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12338,6 +12748,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12368,6 +12780,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12401,6 +12815,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_P stack_pointer[-1] = value2; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12436,6 +12852,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12471,6 +12889,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12523,6 +12943,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12589,6 +13011,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12652,6 +13076,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12702,6 +13128,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12781,6 +13209,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_ stack_pointer = _PyFrame_GetStackPointer(frame); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12817,6 +13247,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS){ stack_pointer[-1] = bottom_out; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12875,6 +13307,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS){ stack_pointer[-1] = res; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12925,6 +13359,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PA stack_pointer[-1] = res; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12962,6 +13398,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS){ STAT_INC(TO_BOOL, hit); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13010,6 +13448,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS){ stack_pointer[-1] = res; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13052,6 +13492,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS){ stack_pointer[-1] = res; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13093,6 +13535,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS){ stack_pointer[-1] = res; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13142,6 +13586,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS){ stack_pointer[-1] = res; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13177,6 +13623,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS){ stack_pointer[-1] = res; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13212,6 +13660,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS) stack_pointer[-1] = res; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13244,6 +13694,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS){ stack_pointer[-1] = res; } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13281,6 +13733,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13342,6 +13796,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13402,6 +13858,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_P assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13453,6 +13911,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13505,6 +13965,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_C assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13575,6 +14037,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13639,6 +14103,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); + start_frame: + TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: From fb1cc3fbf84a377e58ab7eb97bd13f1baac03981 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 31 Jan 2025 20:50:51 +0800 Subject: [PATCH 088/110] Rename tail_call_shim to tail_call_entry --- Python/bytecodes.c | 4 ++-- Python/ceval.c | 4 ++-- Python/generated_cases.c.h | 4 ++-- Python/generated_tail_call_handlers.c.h | 6 +++--- Tools/cases_generator/tier1_tail_call_generator.py | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5b7fdeb6b7fc75..c27a77f4d4321d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5210,7 +5210,7 @@ dummy_func( } #endif #if defined(Py_TAIL_CALL_INTERP) && !defined(IN_TAIL_CALL_INTERP) - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0); + return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); #else DISPATCH(); #endif @@ -5313,7 +5313,7 @@ dummy_func( // If we are in a tail call handler, we want to tail call (DISPATCH). // If we're not then we need the shim frame. #if defined(Py_TAIL_CALL_INTERP) && !defined(IN_TAIL_CALL_INTERP) - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0); + return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); #else DISPATCH(); #endif diff --git a/Python/ceval.c b/Python/ceval.c index 0e25a06517132f..7a6558354ecaf6 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -770,7 +770,7 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) #ifdef Py_TAIL_CALL_INTERP #include "generated_tail_call_handlers.c.h" static inline PyObject * -_TAIL_CALL_shim(TAIL_CALL_PARAMS) +_TAIL_CALL_entry(TAIL_CALL_PARAMS) { opcode = next_instr->op.code; oparg = next_instr->op.arg; @@ -882,7 +882,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif #ifdef Py_TAIL_CALL_INTERP - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0); + return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); #else DISPATCH(); #endif diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index cadebc31754f5e..c0ea966bc2f905 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -9513,7 +9513,7 @@ } #endif #if defined(Py_TAIL_CALL_INTERP) && !defined(IN_TAIL_CALL_INTERP) - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0); + return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); #else DISPATCH(); #endif @@ -9618,7 +9618,7 @@ // If we are in a tail call handler, we want to tail call (DISPATCH). // If we're not then we need the shim frame. #if defined(Py_TAIL_CALL_INTERP) && !defined(IN_TAIL_CALL_INTERP) - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0); + return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); #else DISPATCH(); #endif diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 9557ca75c37bb4..5c16d132689407 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -8,7 +8,7 @@ #endif #define TIER_ONE 1 #define IN_TAIL_CALL_INTERP 1 -static inline PyObject *_TAIL_CALL_shim(TAIL_CALL_PARAMS); +static inline PyObject *_TAIL_CALL_entry(TAIL_CALL_PARAMS); static py_tail_call_funcptr INSTRUCTION_TABLE[256]; Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); @@ -36,7 +36,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS) } #endif #if defined(Py_TAIL_CALL_INTERP) && !defined(IN_TAIL_CALL_INTERP) - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0); + return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); #else DISPATCH(); #endif @@ -141,7 +141,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAM // If we are in a tail call handler, we want to tail call (DISPATCH). // If we're not then we need the shim frame. #if defined(Py_TAIL_CALL_INTERP) && !defined(IN_TAIL_CALL_INTERP) - return _TAIL_CALL_shim(frame, stack_pointer, tstate, next_instr, 0, 0); + return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); #else DISPATCH(); #endif diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index 1a56df6b1bfa50..c1b28b34b2f8a4 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -165,7 +165,7 @@ def generate_tier1( """ ) out = CWriter(outfile, 0, lines) - out.emit("static inline PyObject *_TAIL_CALL_shim(TAIL_CALL_PARAMS);\n") + out.emit("static inline PyObject *_TAIL_CALL_entry(TAIL_CALL_PARAMS);\n") out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256];\n"); generate_label_handlers(analysis, outfile, lines) From 1dda505ea0e1db91e01c014b701074256e6fe026 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 31 Jan 2025 21:05:35 +0800 Subject: [PATCH 089/110] cleanup --- Python/bytecodes.c | 4 ++++ Python/ceval.c | 20 +------------------- Python/generated_cases.c.h | 4 ++++ Python/generated_tail_call_handlers.c.h | 4 ++++ 4 files changed, 13 insertions(+), 19 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c27a77f4d4321d..43816d71a42c81 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5208,6 +5208,10 @@ dummy_func( if (lltrace < 0) { goto exit_unwind; } + /* _PyEval_EvalFrameDefault() must not be called with an exception set, + because it can clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!_PyErr_Occurred(tstate)); #endif #if defined(Py_TAIL_CALL_INTERP) && !defined(IN_TAIL_CALL_INTERP) return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); diff --git a/Python/ceval.c b/Python/ceval.c index 7a6558354ecaf6..6ff38ccfa2d25a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -865,27 +865,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int const _PyUOpInstruction *next_uop = NULL; #endif - if (_Py_EnterRecursivePy(tstate)) { - goto exit_unwind; - } - - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - LLTRACE_RESUME_FRAME(); - -#ifdef Py_DEBUG - /* _PyEval_EvalFrameDefault() must not be called with an exception set, - because it can clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!_PyErr_Occurred(tstate)); -#endif + goto start_frame; -#ifdef Py_TAIL_CALL_INTERP - return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); -#else - DISPATCH(); -#endif #include "generated_cases.c.h" diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c0ea966bc2f905..067ec188f4f2f5 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -9511,6 +9511,10 @@ if (lltrace < 0) { goto exit_unwind; } + /* _PyEval_EvalFrameDefault() must not be called with an exception set, + because it can clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!_PyErr_Occurred(tstate)); #endif #if defined(Py_TAIL_CALL_INTERP) && !defined(IN_TAIL_CALL_INTERP) return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 5c16d132689407..0da606497aefc3 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -34,6 +34,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS) if (lltrace < 0) { TAIL_CALL(exit_unwind); } + /* _PyEval_EvalFrameDefault() must not be called with an exception set, + because it can clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!_PyErr_Occurred(tstate)); #endif #if defined(Py_TAIL_CALL_INTERP) && !defined(IN_TAIL_CALL_INTERP) return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); From 24df49fdb0ff778bafa71eecc19900a2fb8b6524 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 1 Feb 2025 01:26:48 +0800 Subject: [PATCH 090/110] Merge upstream changes, remove complicated hack --- Python/bytecodes.c | 11 - Python/ceval.c | 4 + Python/generated_cases.c.h | 54 +- Python/generated_tail_call_handlers.c.h | 479 +----------------- .../tier1_tail_call_generator.py | 3 +- 5 files changed, 27 insertions(+), 524 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d440ca52241bc7..7b4038c7b75a00 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5213,11 +5213,7 @@ dummy_func( caller loses its exception */ assert(!_PyErr_Occurred(tstate)); #endif -#if defined(Py_TAIL_CALL_INTERP) && !defined(IN_TAIL_CALL_INTERP) - return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); -#else DISPATCH(); -#endif } label(pop_4_error) { @@ -5313,14 +5309,7 @@ dummy_func( lltrace_resume_frame(frame); } #endif - // This is a little complicated... - // If we are in a tail call handler, we want to tail call (DISPATCH). - // If we're not then we need the shim frame. -#if defined(Py_TAIL_CALL_INTERP) && !defined(IN_TAIL_CALL_INTERP) - return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); -#else DISPATCH(); -#endif } label(exit_unwind) { diff --git a/Python/ceval.c b/Python/ceval.c index 9ac5d87babdc59..54a1bed8f91243 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -865,7 +865,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int const _PyUOpInstruction *next_uop = NULL; #endif +#ifdef Py_TAIL_CALL_INTERP + return _TAIL_CALL_start_frame(frame, stack_pointer, tstate, next_instr, 0, 0); +#else goto start_frame; +#endif #include "generated_cases.c.h" diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 5edfbdf6aaf856..af9c44ed02e1c8 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -9505,22 +9505,24 @@ } next_instr = frame->instr_ptr; stack_pointer = _PyFrame_GetStackPointer(frame); - #ifdef Py_DEBUG - int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); - frame->lltrace = lltrace; - if (lltrace < 0) { - goto exit_unwind; + #ifdef LLTRACE + { + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + frame->lltrace = lltrace; + if (lltrace < 0) { + goto exit_unwind; + } } + #endif + + #ifdef Py_DEBUG /* _PyEval_EvalFrameDefault() must not be called with an exception set, because it can clear it (directly or indirectly) and so the caller loses its exception */ assert(!_PyErr_Occurred(tstate)); #endif - #if defined(Py_TAIL_CALL_INTERP) && !defined(IN_TAIL_CALL_INTERP) - return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); - #else + DISPATCH(); - #endif } pop_4_error: @@ -9618,14 +9620,7 @@ lltrace_resume_frame(frame); } #endif - // This is a little complicated... - // If we are in a tail call handler, we want to tail call (DISPATCH). - // If we're not then we need the shim frame. - #if defined(Py_TAIL_CALL_INTERP) && !defined(IN_TAIL_CALL_INTERP) - return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); - #else DISPATCH(); - #endif } exit_unwind: @@ -9649,32 +9644,5 @@ goto error; } - start_frame: - { - if (_Py_EnterRecursivePy(tstate)) { - goto exit_unwind; - } - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - #ifdef LLTRACE - { - int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); - frame->lltrace = lltrace; - if (lltrace < 0) { - goto exit_unwind; - } - } - #endif - - #ifdef Py_DEBUG - /* _PyEval_EvalFrameDefault() must not be called with an exception set, - because it can clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!_PyErr_Occurred(tstate)); - #endif - - DISPATCH(); - } - /* END LABELS */ #undef TIER_ONE diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 0da606497aefc3..ffd74d87f1c104 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -7,7 +7,6 @@ #error "This file is for tail-calling interpreter only." #endif #define TIER_ONE 1 -#define IN_TAIL_CALL_INTERP 1 static inline PyObject *_TAIL_CALL_entry(TAIL_CALL_PARAMS); static py_tail_call_funcptr INSTRUCTION_TABLE[256]; @@ -19,7 +18,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_resume_with_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS) { @@ -28,22 +26,24 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS) } next_instr = frame->instr_ptr; stack_pointer = _PyFrame_GetStackPointer(frame); - #ifdef Py_DEBUG - int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); - frame->lltrace = lltrace; - if (lltrace < 0) { - TAIL_CALL(exit_unwind); + #ifdef LLTRACE + { + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + frame->lltrace = lltrace; + if (lltrace < 0) { + TAIL_CALL(exit_unwind); + } } + #endif + + #ifdef Py_DEBUG /* _PyEval_EvalFrameDefault() must not be called with an exception set, because it can clear it (directly or indirectly) and so the caller loses its exception */ assert(!_PyErr_Occurred(tstate)); #endif - #if defined(Py_TAIL_CALL_INTERP) && !defined(IN_TAIL_CALL_INTERP) - return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); - #else + DISPATCH(); - #endif } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_4_error(TAIL_CALL_PARAMS) @@ -141,14 +141,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAM lltrace_resume_frame(frame); } #endif - // This is a little complicated... - // If we are in a tail call handler, we want to tail call (DISPATCH). - // If we're not then we need the shim frame. - #if defined(Py_TAIL_CALL_INTERP) && !defined(IN_TAIL_CALL_INTERP) - return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); - #else DISPATCH(); - #endif } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS) @@ -167,11 +160,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS) tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return NULL; } - TAIL_CALL(resume_with_error); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_resume_with_error(TAIL_CALL_PARAMS) -{ next_instr = frame->instr_ptr; stack_pointer = _PyFrame_GetStackPointer(frame); TAIL_CALL(error); @@ -245,8 +233,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS){ @@ -312,8 +298,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS){ @@ -378,8 +362,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS){ @@ -444,8 +426,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS){ @@ -515,8 +495,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS){ @@ -611,8 +589,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS){ @@ -678,8 +654,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS){ @@ -744,8 +718,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS){ @@ -811,8 +783,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS){ @@ -877,8 +847,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS){ @@ -947,8 +915,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS){ @@ -1014,8 +980,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PARAMS){ @@ -1074,8 +1038,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PAR TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_PARAMS){ @@ -1174,8 +1136,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL_PARAMS){ @@ -1255,8 +1215,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_PARAMS){ @@ -1328,8 +1286,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS){ @@ -1397,8 +1353,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CAL TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS){ @@ -1435,8 +1389,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS){ @@ -1495,8 +1447,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS){ @@ -1561,8 +1511,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS){ @@ -1607,8 +1555,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS){ @@ -1662,8 +1608,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS){ @@ -1700,8 +1644,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS){ @@ -1729,8 +1671,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS){ @@ -1910,8 +1850,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS){ @@ -2049,8 +1987,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_C TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS){ @@ -2205,8 +2141,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS){ @@ -2345,8 +2279,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_C TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS){ @@ -2452,8 +2384,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS){ @@ -2565,8 +2495,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS){ @@ -2679,8 +2607,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS( TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS){ @@ -2784,8 +2710,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS){ @@ -2967,8 +2891,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS){ @@ -3005,8 +2927,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS){ @@ -3050,8 +2970,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS){ @@ -3126,8 +3044,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ @@ -3304,8 +3220,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS){ @@ -3451,8 +3365,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_P TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS){ @@ -3574,8 +3486,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ @@ -3692,8 +3602,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ @@ -3768,8 +3676,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ @@ -3839,8 +3745,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS){ @@ -3958,8 +3862,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS){ @@ -4077,8 +3979,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS){ @@ -4190,8 +4090,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS){ @@ -4306,8 +4204,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS){ @@ -4423,8 +4319,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS){ @@ -4549,8 +4443,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PAR TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS){ @@ -4660,8 +4552,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS){ @@ -4740,8 +4630,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ @@ -4820,8 +4708,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS){ @@ -4877,8 +4763,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS){ @@ -4944,8 +4828,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS){ @@ -4992,8 +4874,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS){ @@ -5054,8 +4934,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS){ @@ -5135,8 +5013,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS){ @@ -5202,8 +5078,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS){ @@ -5281,8 +5155,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS){ @@ -5349,8 +5221,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS){ @@ -5415,8 +5285,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS){ @@ -5468,8 +5336,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS){ @@ -5522,8 +5388,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS){ @@ -5562,8 +5426,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS){ @@ -5597,8 +5459,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS){ @@ -5635,8 +5495,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS){ @@ -5672,8 +5530,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS){ @@ -5712,8 +5568,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS){ @@ -5750,8 +5604,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS){ @@ -5792,8 +5644,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS){ @@ -5841,8 +5691,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS){ @@ -5882,8 +5730,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS){ @@ -5931,8 +5777,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS){ @@ -5984,8 +5828,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS){ @@ -6035,8 +5877,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS){ @@ -6071,8 +5911,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS){ @@ -6109,8 +5947,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS){ @@ -6162,8 +5998,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS){ @@ -6202,8 +6036,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS){ @@ -6233,8 +6065,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS){ @@ -6278,8 +6108,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS){ @@ -6320,8 +6148,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS){ @@ -6402,8 +6228,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS){ @@ -6483,8 +6307,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS){ @@ -6559,8 +6381,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS){ @@ -6627,8 +6447,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS){ @@ -6700,8 +6518,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS){ @@ -6767,8 +6583,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS){ @@ -6807,8 +6621,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS){ @@ -6844,8 +6656,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS){ @@ -6882,8 +6692,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS){ @@ -6923,8 +6731,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS){ @@ -6986,8 +6792,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS){ @@ -7025,8 +6829,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS){ @@ -7070,8 +6872,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS){ @@ -7259,8 +7059,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS){ @@ -7287,8 +7085,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS){ @@ -7331,8 +7127,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_P TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS){ @@ -7376,8 +7170,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_P TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS){ @@ -7423,8 +7215,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS){ @@ -7480,8 +7270,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS){ @@ -7519,8 +7307,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS){ @@ -7563,8 +7349,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS){ @@ -7592,8 +7376,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_C TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS){ @@ -7650,8 +7432,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS){ @@ -7682,8 +7462,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAI TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS){ @@ -7713,8 +7491,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS){ @@ -7748,8 +7524,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS){ @@ -7784,8 +7558,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(T TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS){ @@ -7822,8 +7594,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS){ @@ -7858,8 +7628,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NON TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS){ @@ -7894,8 +7662,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS){ @@ -7986,8 +7752,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS){ @@ -8049,8 +7813,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_C TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS){ @@ -8136,8 +7898,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS){ @@ -8175,8 +7935,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS){ @@ -8214,8 +7972,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS){ @@ -8277,8 +8033,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS){ @@ -8361,8 +8115,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS){ @@ -8395,8 +8147,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS){ @@ -8445,8 +8195,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_P TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS){ @@ -8473,8 +8221,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ @@ -8509,8 +8255,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS){ @@ -8566,8 +8310,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ @@ -8662,8 +8404,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS){ @@ -8729,8 +8469,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS){ @@ -8806,8 +8544,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_C TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS){ @@ -8879,8 +8615,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDE TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS){ @@ -8969,8 +8703,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS){ @@ -9041,8 +8773,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS){ @@ -9102,8 +8832,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS){ @@ -9185,8 +8913,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAI TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS){ @@ -9277,8 +9003,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS){ @@ -9333,8 +9057,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT( TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS){ @@ -9410,8 +9132,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VAL TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS){ @@ -9520,8 +9240,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS){ @@ -9598,8 +9316,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS){ @@ -9717,8 +9433,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS){ @@ -9761,8 +9475,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS){ @@ -9803,8 +9515,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_P TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS){ @@ -9856,8 +9566,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_PARAMS){ @@ -9891,8 +9599,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_PA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_MORTAL(TAIL_CALL_PARAMS){ @@ -9925,8 +9631,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_MORTAL(TAIL_CALL_PARA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS){ @@ -9965,8 +9669,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS){ @@ -9998,8 +9700,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS){ @@ -10032,8 +9732,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS){ @@ -10074,8 +9772,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS){ @@ -10111,8 +9807,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS){ @@ -10166,8 +9860,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CAL TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS){ @@ -10256,8 +9948,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_C TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS){ @@ -10324,8 +10014,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS){ @@ -10423,8 +10111,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS){ @@ -10506,8 +10192,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PAR TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS){ @@ -10546,8 +10230,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS){ @@ -10583,8 +10265,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS){ @@ -10617,8 +10297,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS){ @@ -10674,8 +10352,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS){ @@ -10796,8 +10472,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS){ @@ -10860,8 +10534,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_P TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS){ @@ -10936,8 +10608,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS){ @@ -10971,8 +10641,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS){ @@ -11012,8 +10680,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS){ @@ -11059,8 +10725,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS){ @@ -11117,8 +10781,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS){ @@ -11159,8 +10821,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS){ @@ -11194,8 +10854,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS){ @@ -11229,8 +10887,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS){ @@ -11256,8 +10912,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS){ @@ -11283,8 +10937,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS){ @@ -11320,8 +10972,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS){ @@ -11352,8 +11002,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS){ @@ -11389,8 +11037,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS){ @@ -11442,8 +11088,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS){ @@ -11495,8 +11139,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_P TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS){ @@ -11532,8 +11174,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS){ @@ -11564,8 +11204,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS){ @@ -11609,8 +11247,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS){ @@ -11641,8 +11277,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS){ @@ -11686,8 +11320,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS){ @@ -11749,8 +11381,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS){ @@ -11778,8 +11408,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS){ @@ -11868,8 +11496,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS){ @@ -11921,8 +11547,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS){ @@ -11974,8 +11598,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS){ @@ -12022,8 +11644,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS){ @@ -12132,8 +11752,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS){ @@ -12217,8 +11835,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS){ @@ -12274,8 +11890,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS){ @@ -12313,8 +11927,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS){ @@ -12357,8 +11969,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS){ @@ -12396,8 +12006,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS){ @@ -12461,8 +12069,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS){ @@ -12552,8 +12158,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_C TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS){ @@ -12618,8 +12222,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS){ @@ -12733,8 +12335,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS){ @@ -12768,8 +12368,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS){ @@ -12800,8 +12398,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS){ @@ -12835,8 +12431,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_P TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS){ @@ -12872,8 +12466,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS){ @@ -12909,8 +12501,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS){ @@ -12963,8 +12553,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS){ @@ -13031,8 +12619,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS){ @@ -13096,8 +12682,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS){ @@ -13148,8 +12732,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS){ @@ -13229,8 +12811,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS){ @@ -13267,8 +12847,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS){ @@ -13327,8 +12905,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS){ @@ -13379,8 +12955,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS){ @@ -13418,8 +12992,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS){ @@ -13468,8 +13040,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS){ @@ -13512,8 +13082,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS){ @@ -13555,8 +13123,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS){ @@ -13606,8 +13172,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS){ @@ -13643,8 +13207,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS){ @@ -13680,8 +13242,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS){ @@ -13714,8 +13274,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS){ @@ -13753,8 +13311,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS){ @@ -13816,8 +13372,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS){ @@ -13878,8 +13432,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_P TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS){ @@ -13931,8 +13483,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS){ @@ -13985,8 +13535,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_C TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS){ @@ -14057,8 +13605,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS){ @@ -14123,8 +13669,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); - resume_with_error: - TAIL_CALL(resume_with_error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS){ @@ -14397,4 +13941,3 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [234] = _TAIL_CALL_UNKNOWN_OPCODE, }; #undef TIER_ONE -#undef IN_TAIL_CALL_INTERP diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index c1b28b34b2f8a4..776acc76afce57 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -36,7 +36,7 @@ DEFAULT_INPUT = ROOT / "Python/bytecodes.c" DEFAULT_OUTPUT = ROOT / "Python/generated_tail_call_handlers.c.h" -FOOTER = "#undef TIER_ONE\n#undef IN_TAIL_CALL_INTERP\n" +FOOTER = "#undef TIER_ONE\n" class TailCallEmitter(Emitter): @@ -161,7 +161,6 @@ def generate_tier1( #error "This file is for tail-calling interpreter only." #endif #define TIER_ONE 1 -#define IN_TAIL_CALL_INTERP 1 """ ) out = CWriter(outfile, 0, lines) From bd97afa752e8775c7bbb0c0785f0bfe07051957b Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 1 Feb 2025 01:57:21 +0800 Subject: [PATCH 091/110] fixup --- Python/ceval.c | 6 +- Python/generated_tail_call_labels.c.h | 107 ++++++++++++++++++ Tools/cases_generator/tier1_generator.py | 21 ++-- .../tier1_tail_call_generator.py | 65 ++++++++--- 4 files changed, 173 insertions(+), 26 deletions(-) create mode 100644 Python/generated_tail_call_labels.c.h diff --git a/Python/ceval.c b/Python/ceval.c index 54a1bed8f91243..756d7971d7ff53 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -866,13 +866,13 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif #ifdef Py_TAIL_CALL_INTERP - return _TAIL_CALL_start_frame(frame, stack_pointer, tstate, next_instr, 0, 0); + return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, 0, 0); +# include "generated_tail_call_labels.c.h" #else goto start_frame; +# include "generated_cases.c.h" #endif -#include "generated_cases.c.h" - #ifdef _Py_TIER2 // Tier 2 is also here! diff --git a/Python/generated_tail_call_labels.c.h b/Python/generated_tail_call_labels.c.h new file mode 100644 index 00000000000000..36ec137f32534f --- /dev/null +++ b/Python/generated_tail_call_labels.c.h @@ -0,0 +1,107 @@ +// This file is generated by Tools/cases_generator/tier1_tail_call_generator.py +// from: +// Python/bytecodes.c +// Do not edit! + +#ifndef Py_TAIL_CALL_INTERP + #error "This file is for tail-calling interpreter only." +#endif +#define TIER_ONE 1 + + error: + { + /* Double-check exception status. */ + #ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } + #else + assert(_PyErr_Occurred(tstate)); + #endif + + /* Log traceback info. */ + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); + goto exception_unwind; + } + + exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ + #ifdef LLTRACE + if (frame->lltrace >= 5) { + lltrace_resume_frame(frame); + } + #endif + _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); + ; + } + + exit_unwind: + { + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + +#undef TIER_ONE diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index fcea7deff85f7f..e4e48011132ecc 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -15,6 +15,7 @@ Flush, analysis_error, StackItem, + Label, ) from generators_common import ( DEFAULT_INPUT, @@ -221,21 +222,21 @@ def generate_tier1( #endif /* Py_TAIL_CALL_INTERP */ {LABEL_START_MARKER} """) - generate_tier1_labels(analysis, outfile, lines) + out = CWriter(outfile, 2, lines) + emitter = Emitter(out) + generate_tier1_labels(analysis.labels, emitter) outfile.write(f"{LABEL_END_MARKER}\n") outfile.write(FOOTER) def generate_tier1_labels( - analysis: Analysis, outfile: TextIO, lines: bool + labels: dict[str, Label], emitter: Emitter ) -> None: - out = CWriter(outfile, 2, lines) - out.emit("\n") - for name, label in analysis.labels.items(): - out.emit(f"{name}:\n") - for tkn in label.body: - out.emit(tkn) - out.emit("\n") - out.emit("\n") + emitter.emit("\n") + for name, label in labels.items(): + emitter.emit(f"{name}:\n") + emitter.emit_label(label) + emitter.emit("\n") + emitter.emit("\n") def generate_tier1_cases( analysis: Analysis, outfile: TextIO, lines: bool diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index 776acc76afce57..42dfbd58a16927 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -26,7 +26,8 @@ ) from tier1_generator import ( - write_single_inst + write_single_inst, + generate_tier1_labels, ) from lexer import Token @@ -35,9 +36,22 @@ DEFAULT_INPUT = ROOT / "Python/bytecodes.c" DEFAULT_OUTPUT = ROOT / "Python/generated_tail_call_handlers.c.h" +DEFAULT_LABELS_OUTPUT = ROOT / "Python/generated_tail_call_labels.c.h" +PRELUDE = """ +#ifndef Py_TAIL_CALL_INTERP + #error "This file is for tail-calling interpreter only." +#endif +#define TIER_ONE 1 +""" FOOTER = "#undef TIER_ONE\n" +NEEDED_LABELS = { + "error", + "exception_unwind", + "exit_unwind", +} + class TailCallEmitter(Emitter): def __init__(self, out: CWriter, analysis: Analysis): @@ -110,11 +124,32 @@ def goto( name = next(tkn_iter) next(tkn_iter) assert name.kind == "IDENTIFIER" - self.out.emit("\n") + self.out.start_line() self.emit(f"TAIL_CALL({name.text});\n") return True +class TailCallCevalLabelsEmitter(Emitter): + def __init__(self, out: CWriter): + super().__init__(out) + self._replacers = { + 'DISPATCH': self.dispatch, + } + + def dispatch( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop | Label, + storage: Storage, + inst: Instruction | None, + ) -> bool: + # Replace DISPATCH with _TAIL_CALL_entry(...) + next(tkn_iter) + next(tkn_iter) + self.emit("_TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0);\n") + return True + def function_proto(name: str) -> str: return f"Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_{name}(TAIL_CALL_PARAMS)" @@ -152,17 +187,17 @@ def uses_this(inst: Instruction) -> bool: return False def generate_tier1( - filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool + filenames: list[str], analysis: Analysis, outfile: TextIO, labels_outfile: TextIO, lines: bool ) -> None: + # Write labels required for main ceval + write_header(__file__, filenames, labels_outfile) + labels_outfile.write(PRELUDE) + out = CWriter(labels_outfile, 2, lines) + emitter = TailCallCevalLabelsEmitter(out) + generate_tier1_labels({label: analysis.labels[label] for label in NEEDED_LABELS}, emitter) + labels_outfile.write(FOOTER) write_header(__file__, filenames, outfile) - outfile.write( - """ -#ifndef Py_TAIL_CALL_INTERP - #error "This file is for tail-calling interpreter only." -#endif -#define TIER_ONE 1 -""" - ) + outfile.write(PRELUDE) out = CWriter(outfile, 0, lines) out.emit("static inline PyObject *_TAIL_CALL_entry(TAIL_CALL_PARAMS);\n") out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256];\n"); @@ -232,6 +267,10 @@ def generate_tier1( "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT ) +arg_parser.add_argument( + "-lo", "--labels-output", type=str, help="Generated labels", default=DEFAULT_LABELS_OUTPUT +) + arg_parser.add_argument( "-l", "--emit-line-directives", help="Emit #line directives", action="store_true" ) @@ -246,5 +285,5 @@ def generate_tier1( if len(args.input) == 0: args.input.append(DEFAULT_INPUT) data = analyze_files(args.input) - with open(args.output, "w") as outfile: - generate_tier1(args.input, data, outfile, args.emit_line_directives) + with open(args.output, "w") as outfile, open(args.labels_output, 'w') as labels_outfile: + generate_tier1(args.input, data, outfile, labels_outfile, args.emit_line_directives) From c6967c6b795d9253011f63f6d647e77042fe31c4 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 1 Feb 2025 01:59:18 +0800 Subject: [PATCH 092/110] add to generated --- .gitattributes | 1 + .github/workflows/tail-call.yml | 2 ++ Tools/c-analyzer/cpython/_parser.py | 1 + 3 files changed, 4 insertions(+) diff --git a/.gitattributes b/.gitattributes index ff479c49ba5394..74d7c7f0bb97f9 100644 --- a/.gitattributes +++ b/.gitattributes @@ -96,6 +96,7 @@ Python/Python-ast.c generated Python/executor_cases.c.h generated Python/generated_cases.c.h generated Python/generated_tail_call_handlers.c.h generated +Python/generated_tail_call_labels.c.h generated Python/optimizer_cases.c.h generated Python/opcode_targets.h generated Python/stdlib_module_names.h generated diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index c05bb1201ff95c..ed4fdea6ca05f4 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -6,12 +6,14 @@ on: - 'Python/ceval.c' - 'Python/ceval_macros.h' - 'Python/generated_tail_call_handlers.c.h' + - 'Python/generated_tail_call_labels.c.h' push: paths: - 'Python/bytecodes.c' - 'Python/ceval.c' - 'Python/ceval_macros.h' - 'Python/generated_tail_call_handlers.c.h' + - 'Python/generated_tail_call_labels.c.h' workflow_dispatch: permissions: diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index b77046c320222f..8fa26758f964c7 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -81,6 +81,7 @@ def clean_lines(text): Python/frozen_modules/*.h Python/generated_cases.c.h Python/generated_tail_call_handlers.c.h +Python/generated_tail_call_labels.c.h Python/executor_cases.c.h Python/optimizer_cases.c.h From 587a433dc8a4c250228ec82b00ca55ecd4ea2b61 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 1 Feb 2025 02:04:57 +0800 Subject: [PATCH 093/110] remove dual definition --- Python/bytecodes.c | 22 - Python/generated_cases.c.h | 54 +- Python/generated_tail_call_handlers.c.h | 944 ++++++++++++------------ Python/generated_tail_call_labels.c.h | 48 +- 4 files changed, 523 insertions(+), 545 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7b4038c7b75a00..d5a4eb23914d2c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5194,28 +5194,6 @@ dummy_func( assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); } - label(start_frame) { - if (_Py_EnterRecursivePy(tstate)) { - goto exit_unwind; - } - - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - -#ifdef Py_DEBUG - int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); - frame->lltrace = lltrace; - if (lltrace < 0) { - goto exit_unwind; - } - /* _PyEval_EvalFrameDefault() must not be called with an exception set, - because it can clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!_PyErr_Occurred(tstate)); -#endif - DISPATCH(); - } - label(pop_4_error) { STACK_SHRINK(1); goto pop_3_error; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index af9c44ed02e1c8..7988f782655953 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -9498,33 +9498,6 @@ #endif /* Py_TAIL_CALL_INTERP */ /* BEGIN LABELS */ - start_frame: - { - if (_Py_EnterRecursivePy(tstate)) { - goto exit_unwind; - } - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - #ifdef LLTRACE - { - int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); - frame->lltrace = lltrace; - if (lltrace < 0) { - goto exit_unwind; - } - } - #endif - - #ifdef Py_DEBUG - /* _PyEval_EvalFrameDefault() must not be called with an exception set, - because it can clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!_PyErr_Occurred(tstate)); - #endif - - DISPATCH(); - } - pop_4_error: { STACK_SHRINK(1); @@ -9644,5 +9617,32 @@ goto error; } + start_frame: + { + if (_Py_EnterRecursivePy(tstate)) { + goto exit_unwind; + } + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + #ifdef LLTRACE + { + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + frame->lltrace = lltrace; + if (lltrace < 0) { + goto exit_unwind; + } + } + #endif + + #ifdef Py_DEBUG + /* _PyEval_EvalFrameDefault() must not be called with an exception set, + because it can clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!_PyErr_Occurred(tstate)); + #endif + + DISPATCH(); + } + /* END LABELS */ #undef TIER_ONE diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index ffd74d87f1c104..bf3aa9ccda127e 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -10,7 +10,6 @@ static inline PyObject *_TAIL_CALL_entry(TAIL_CALL_PARAMS); static py_tail_call_funcptr INSTRUCTION_TABLE[256]; -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_4_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_3_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); @@ -18,33 +17,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS) -{ - if (_Py_EnterRecursivePy(tstate)) { - TAIL_CALL(exit_unwind); - } - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - #ifdef LLTRACE - { - int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); - frame->lltrace = lltrace; - if (lltrace < 0) { - TAIL_CALL(exit_unwind); - } - } - #endif - - #ifdef Py_DEBUG - /* _PyEval_EvalFrameDefault() must not be called with an exception set, - because it can clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!_PyErr_Occurred(tstate)); - #endif - - DISPATCH(); -} +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_4_error(TAIL_CALL_PARAMS) { @@ -165,6 +138,33 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS) TAIL_CALL(error); } +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS) +{ + if (_Py_EnterRecursivePy(tstate)) { + TAIL_CALL(exit_unwind); + } + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + #ifdef LLTRACE + { + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + frame->lltrace = lltrace; + if (lltrace < 0) { + TAIL_CALL(exit_unwind); + } + } + #endif + + #ifdef Py_DEBUG + /* _PyEval_EvalFrameDefault() must not be called with an exception set, + because it can clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!_PyErr_Occurred(tstate)); + #endif + + DISPATCH(); +} + Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS){ @@ -217,8 +217,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -233,6 +231,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS){ @@ -282,8 +282,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -298,6 +296,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS){ @@ -346,8 +346,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -362,6 +360,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS){ @@ -410,8 +410,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -426,6 +424,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS){ @@ -479,8 +479,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -495,6 +493,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS){ @@ -573,8 +573,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -589,6 +587,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS){ @@ -638,8 +638,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -654,6 +652,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS){ @@ -702,8 +702,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -718,6 +716,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS){ @@ -767,8 +767,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -783,6 +781,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS){ @@ -831,8 +831,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -847,6 +845,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS){ @@ -899,8 +899,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -915,6 +913,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS){ @@ -964,8 +964,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -980,6 +978,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PARAMS){ @@ -1022,8 +1022,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PAR assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1038,6 +1036,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PAR TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_PARAMS){ @@ -1120,8 +1120,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_ } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1136,6 +1134,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL_PARAMS){ @@ -1199,8 +1199,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1215,6 +1213,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_PARAMS){ @@ -1270,8 +1270,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1286,6 +1284,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS){ @@ -1337,8 +1337,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CAL assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1353,6 +1351,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CAL TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS){ @@ -1373,8 +1373,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1389,6 +1387,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS){ @@ -1431,8 +1431,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1447,6 +1445,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS){ @@ -1495,8 +1495,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1511,6 +1509,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS){ @@ -1539,8 +1539,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1555,6 +1553,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS){ @@ -1592,8 +1592,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1608,6 +1606,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS){ @@ -1628,8 +1628,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1644,6 +1642,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS){ @@ -1655,8 +1655,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS){ Py_FatalError("Executing a cache."); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1671,6 +1669,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS){ @@ -1834,8 +1834,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1850,6 +1848,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS){ @@ -1971,8 +1971,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_C } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -1987,6 +1985,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_C TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS){ @@ -2125,8 +2125,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -2141,6 +2139,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS){ @@ -2263,8 +2263,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_C } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -2279,6 +2277,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_C TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS){ @@ -2368,8 +2368,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -2384,6 +2382,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS){ @@ -2479,8 +2479,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -2495,6 +2493,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS){ @@ -2591,8 +2591,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS( assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -2607,6 +2605,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS( TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS){ @@ -2694,8 +2694,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS) assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -2710,6 +2708,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS){ @@ -2875,8 +2875,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -2891,6 +2889,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS){ @@ -2911,8 +2911,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAM stack_pointer[-1] = res; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -2927,6 +2925,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS){ @@ -2954,8 +2954,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -2970,6 +2968,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS){ @@ -3028,8 +3028,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -3044,6 +3042,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ @@ -3204,8 +3204,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -3220,6 +3218,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS){ @@ -3349,8 +3349,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_P } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -3365,6 +3363,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_P TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS){ @@ -3470,8 +3470,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -3486,6 +3484,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ @@ -3586,8 +3586,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -3602,6 +3600,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ @@ -3660,8 +3660,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -3676,6 +3674,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ @@ -3729,8 +3729,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -3745,6 +3743,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS){ @@ -3846,8 +3846,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -3862,6 +3860,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS){ @@ -3963,8 +3963,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -3979,6 +3977,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS){ @@ -4074,8 +4074,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -4090,6 +4088,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS){ @@ -4188,8 +4188,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -4204,6 +4202,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS){ @@ -4303,8 +4303,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -4319,6 +4317,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS){ @@ -4427,8 +4427,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PAR } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -4443,6 +4441,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PAR TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS){ @@ -4536,8 +4536,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -4552,6 +4550,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS){ @@ -4614,8 +4614,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -4630,6 +4628,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ @@ -4692,8 +4692,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -4708,6 +4706,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS){ @@ -4747,8 +4747,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -4763,6 +4761,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS){ @@ -4812,8 +4812,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS) stack_pointer[-1] = match; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -4828,6 +4826,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS){ @@ -4858,8 +4858,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS stack_pointer[-1] = b; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -4874,6 +4872,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS){ @@ -4918,8 +4918,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -4934,6 +4932,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS){ @@ -4997,8 +4997,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5013,6 +5011,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS){ @@ -5062,8 +5062,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5078,6 +5076,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS){ @@ -5139,8 +5139,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS) assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5155,6 +5153,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS){ @@ -5205,8 +5205,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS) assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5221,6 +5219,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS){ @@ -5269,8 +5269,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5285,6 +5283,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS){ @@ -5320,8 +5320,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5336,6 +5334,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS){ @@ -5372,8 +5372,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5388,6 +5386,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS){ @@ -5410,8 +5410,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS){ stack_pointer[-1] = result; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5426,6 +5424,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS){ @@ -5443,8 +5443,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5459,6 +5457,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS){ @@ -5479,8 +5479,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS) } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5495,6 +5493,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS){ @@ -5514,8 +5514,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5530,6 +5528,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS){ @@ -5552,8 +5552,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS){ stack_pointer = _PyFrame_GetStackPointer(frame); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5568,6 +5566,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS){ @@ -5588,8 +5588,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS){ SETLOCAL(oparg, PyStackRef_NULL); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5604,6 +5602,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS){ @@ -5628,8 +5628,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS){ } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5644,6 +5642,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS){ @@ -5675,8 +5675,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS){ } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5691,6 +5689,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS){ @@ -5714,8 +5714,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5730,6 +5728,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS){ @@ -5761,8 +5761,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5777,6 +5775,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS){ @@ -5812,8 +5812,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5828,6 +5826,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS){ @@ -5861,8 +5861,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5877,6 +5875,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS){ @@ -5895,8 +5895,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5911,6 +5909,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS){ @@ -5931,8 +5931,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5947,6 +5945,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS){ @@ -5982,8 +5982,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS) #endif /* _Py_TIER2 */ } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -5998,6 +5996,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS){ @@ -6020,8 +6020,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6036,6 +6034,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS){ @@ -6049,8 +6049,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS){ PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); } - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6065,6 +6063,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS){ @@ -6092,8 +6092,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS){ stack_pointer[-1] = res; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6108,6 +6106,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS){ @@ -6132,8 +6132,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6148,6 +6146,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS){ @@ -6212,8 +6212,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6228,6 +6226,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS){ @@ -6291,8 +6291,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS){ } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6307,6 +6305,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS){ @@ -6365,8 +6365,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6381,6 +6379,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS){ @@ -6431,8 +6431,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS) assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6447,6 +6445,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS){ @@ -6502,8 +6502,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS) assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6518,6 +6516,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS){ @@ -6567,8 +6567,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS){ stack_pointer[-1] = iter; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6583,6 +6581,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS){ @@ -6605,8 +6605,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6621,6 +6619,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS){ @@ -6640,8 +6640,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS){ stack_pointer[-1] = iter; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6656,6 +6654,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS){ @@ -6676,8 +6676,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS){ stack_pointer[-1] = iter; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6692,6 +6690,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS){ @@ -6715,8 +6715,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6731,6 +6729,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS){ @@ -6776,8 +6776,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PA stack_pointer[-1] = iter; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6792,6 +6790,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS){ @@ -6813,8 +6813,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6829,6 +6827,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS){ @@ -6856,8 +6856,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -6872,6 +6870,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS){ @@ -7043,8 +7043,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7059,6 +7057,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS){ @@ -7069,8 +7069,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TA INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); Py_MUSTTAIL return (INSTRUCTION_TABLE[CALL_FUNCTION_EX])(frame, stack_pointer, tstate, this_instr, opcode, oparg); } - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7085,6 +7083,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS){ @@ -7111,8 +7111,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_P PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); Py_MUSTTAIL return (INSTRUCTION_TABLE[CALL_KW])(frame, stack_pointer, tstate, this_instr, opcode, oparg); } - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7127,6 +7125,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_P TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS){ @@ -7154,8 +7154,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_P assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7170,6 +7168,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_P TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS){ @@ -7199,8 +7199,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7215,6 +7213,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS){ @@ -7254,8 +7254,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_ } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7270,6 +7268,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS){ @@ -7291,8 +7291,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CA opcode = next_opcode; DISPATCH_GOTO(); } - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7307,6 +7305,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS){ @@ -7333,8 +7333,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_ } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7349,6 +7347,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS){ @@ -7360,8 +7360,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_C INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7376,6 +7374,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_C TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS){ @@ -7416,8 +7416,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARA opcode = original_opcode; DISPATCH_GOTO(); } - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7432,6 +7430,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS){ @@ -7446,8 +7446,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAI PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); Py_MUSTTAIL return (INSTRUCTION_TABLE[LOAD_SUPER_ATTR])(frame, stack_pointer, tstate, this_instr, opcode, oparg); } - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7462,6 +7460,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAI TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS){ @@ -7475,8 +7475,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7491,6 +7489,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS){ @@ -7508,8 +7508,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7524,6 +7522,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS){ @@ -7542,8 +7542,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(T } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7558,6 +7556,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(T TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS){ @@ -7578,8 +7578,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TA } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7594,6 +7592,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS){ @@ -7612,8 +7612,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NON } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7628,6 +7626,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NON TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS){ @@ -7646,8 +7646,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TA } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7662,6 +7660,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS){ @@ -7736,8 +7736,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PA } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7752,6 +7750,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS){ @@ -7797,8 +7797,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_C assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7813,6 +7811,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_C TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS){ @@ -7882,8 +7882,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7898,6 +7896,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS){ @@ -7919,8 +7919,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAM /* Not strictly necessary, but prevents warnings */ return result; } - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7935,6 +7933,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS){ @@ -7956,8 +7956,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7972,6 +7970,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS){ @@ -8017,8 +8017,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS){ } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8033,6 +8031,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS){ @@ -8099,8 +8099,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARA } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8115,6 +8113,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS){ @@ -8131,8 +8131,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_ JUMPBY(-oparg); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8147,6 +8145,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS){ @@ -8179,8 +8179,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_P } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8195,6 +8193,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_P TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS){ @@ -8205,8 +8205,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS){ JUMPBY(oparg); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8221,6 +8219,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ @@ -8239,8 +8239,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8255,6 +8253,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS){ @@ -8294,8 +8294,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8310,6 +8308,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ @@ -8388,8 +8388,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8404,6 +8402,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS){ @@ -8453,8 +8453,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8469,6 +8467,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS){ @@ -8528,8 +8528,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_C assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8544,6 +8542,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_C TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS){ @@ -8599,8 +8599,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDE frame->return_offset = 10 ; DISPATCH_INLINED(new_frame); } - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8615,6 +8613,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDE TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS){ @@ -8687,8 +8687,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8703,6 +8701,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS){ @@ -8757,8 +8757,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8773,6 +8771,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS){ @@ -8816,8 +8816,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8832,6 +8830,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS){ @@ -8897,8 +8897,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAI assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -8913,6 +8911,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAI TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS){ @@ -8987,8 +8987,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9003,6 +9001,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS){ @@ -9041,8 +9041,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT( stack_pointer[-1] = attr; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9057,6 +9055,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT( TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS){ @@ -9116,8 +9116,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VAL stack_pointer[-1] = attr; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9132,6 +9130,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VAL TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS){ @@ -9224,8 +9224,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9240,6 +9238,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS){ @@ -9300,8 +9300,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9316,6 +9314,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS){ @@ -9417,8 +9417,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9433,6 +9431,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS){ @@ -9459,8 +9459,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9475,6 +9473,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS){ @@ -9499,8 +9499,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_P assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9515,6 +9513,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_P TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS){ @@ -9550,8 +9550,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9566,6 +9564,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_PARAMS){ @@ -9583,8 +9583,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_PA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9599,6 +9597,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_PA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_MORTAL(TAIL_CALL_PARAMS){ @@ -9615,8 +9615,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_MORTAL(TAIL_CALL_PARA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9631,6 +9629,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_MORTAL(TAIL_CALL_PARA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS){ @@ -9653,8 +9653,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9669,6 +9667,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS){ @@ -9684,8 +9684,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9700,6 +9698,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS){ @@ -9716,8 +9716,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9732,6 +9730,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS){ @@ -9756,8 +9756,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9772,6 +9770,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS){ @@ -9791,8 +9791,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9807,6 +9805,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS){ @@ -9844,8 +9844,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CAL stack_pointer[-1] = value; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9860,6 +9858,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CAL TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS){ @@ -9932,8 +9932,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_C stack_pointer[-1] = v; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -9948,6 +9946,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_C TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS){ @@ -9998,8 +9998,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10014,6 +10012,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS){ @@ -10095,8 +10095,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10111,6 +10109,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS){ @@ -10176,8 +10176,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PAR assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10192,6 +10190,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PAR TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS){ @@ -10214,8 +10214,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10230,6 +10228,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS){ @@ -10249,8 +10249,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10265,6 +10263,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS){ @@ -10281,8 +10281,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS) assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10297,6 +10295,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS){ @@ -10336,8 +10336,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10352,6 +10350,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS){ @@ -10456,8 +10456,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10472,6 +10470,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS){ @@ -10518,8 +10518,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_P assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10534,6 +10532,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_P TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS){ @@ -10592,8 +10592,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10608,6 +10606,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS){ @@ -10625,8 +10625,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS){ SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell)); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10641,6 +10639,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS){ @@ -10664,8 +10664,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS){ stack_pointer[-1] = func; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10680,6 +10678,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS){ @@ -10709,8 +10709,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10725,6 +10723,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS){ @@ -10765,8 +10765,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10781,6 +10779,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS){ @@ -10805,8 +10805,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10821,6 +10819,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS){ @@ -10838,8 +10838,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10854,6 +10852,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS){ @@ -10871,8 +10871,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS) assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10887,6 +10885,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS){ @@ -10896,8 +10896,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS){ INSTRUCTION_STATS(NOP); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10912,6 +10910,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS){ @@ -10921,8 +10921,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS){ INSTRUCTION_STATS(NOT_TAKEN); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10937,6 +10935,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS){ @@ -10956,8 +10956,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -10972,6 +10970,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS){ @@ -10986,8 +10986,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11002,6 +11000,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS){ @@ -11021,8 +11021,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11037,6 +11035,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS){ @@ -11072,8 +11072,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11088,6 +11086,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS){ @@ -11123,8 +11123,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_P assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11139,6 +11137,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_P TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS){ @@ -11158,8 +11158,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11174,6 +11172,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS){ @@ -11188,8 +11188,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11204,6 +11202,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS){ @@ -11231,8 +11231,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11247,6 +11245,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS){ @@ -11261,8 +11261,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11277,6 +11275,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS){ @@ -11304,8 +11304,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS){ } goto error; } - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11320,6 +11318,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS){ @@ -11365,8 +11365,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS){ stack_pointer = _PyFrame_GetStackPointer(frame); goto exception_unwind; } - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11381,6 +11379,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS){ @@ -11392,8 +11392,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS){ Py_FatalError("Executing RESERVED instruction."); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11408,6 +11406,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS){ @@ -11480,8 +11480,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS){ } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11496,6 +11494,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS){ @@ -11531,8 +11531,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS){ #endif } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11547,6 +11545,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS){ @@ -11582,8 +11582,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAM assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11598,6 +11596,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAM TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS){ @@ -11628,8 +11628,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11644,6 +11642,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS){ @@ -11736,8 +11736,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS){ stack_pointer[-1] = retval; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11752,6 +11750,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS){ @@ -11819,8 +11819,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS){ } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11835,6 +11833,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS){ @@ -11874,8 +11874,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARA } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11890,6 +11888,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS){ @@ -11911,8 +11911,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11927,6 +11925,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS){ @@ -11953,8 +11953,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -11969,6 +11967,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS){ @@ -11990,8 +11990,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12006,6 +12004,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS){ @@ -12053,8 +12053,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12069,6 +12067,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS){ @@ -12142,8 +12142,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_C } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12158,6 +12156,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_C TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS){ @@ -12206,8 +12206,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12222,6 +12220,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS){ @@ -12319,8 +12319,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P } } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12335,6 +12333,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS){ @@ -12352,8 +12352,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12368,6 +12366,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS){ @@ -12382,8 +12382,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12398,6 +12396,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS){ @@ -12415,8 +12415,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_P stack_pointer[-1] = value2; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12431,6 +12429,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_P TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS){ @@ -12450,8 +12450,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12466,6 +12464,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS){ @@ -12485,8 +12485,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12501,6 +12499,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS){ @@ -12537,8 +12537,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12553,6 +12551,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS){ @@ -12603,8 +12603,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12619,6 +12617,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS){ @@ -12666,8 +12666,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12682,6 +12680,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS){ @@ -12716,8 +12716,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12732,6 +12730,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS){ @@ -12795,8 +12795,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_ stack_pointer = _PyFrame_GetStackPointer(frame); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12811,6 +12809,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS){ @@ -12831,8 +12831,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS){ stack_pointer[-1] = bottom_out; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12847,6 +12845,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS){ @@ -12889,8 +12889,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS){ stack_pointer[-1] = res; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12905,6 +12903,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS){ @@ -12939,8 +12939,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PA stack_pointer[-1] = res; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12955,6 +12953,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS){ @@ -12976,8 +12976,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS){ STAT_INC(TO_BOOL, hit); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -12992,6 +12990,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS){ @@ -13024,8 +13024,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS){ stack_pointer[-1] = res; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13040,6 +13038,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS){ @@ -13066,8 +13066,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS){ stack_pointer[-1] = res; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13082,6 +13080,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS){ @@ -13107,8 +13107,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS){ stack_pointer[-1] = res; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13123,6 +13121,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS){ @@ -13156,8 +13156,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS){ stack_pointer[-1] = res; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13172,6 +13170,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS){ @@ -13191,8 +13191,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS){ stack_pointer[-1] = res; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13207,6 +13205,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS){ @@ -13226,8 +13226,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS) stack_pointer[-1] = res; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13242,6 +13240,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS) TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS){ @@ -13258,8 +13258,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS){ stack_pointer[-1] = res; } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13274,6 +13272,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS){ @@ -13295,8 +13295,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13311,6 +13309,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS){ @@ -13356,8 +13356,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13372,6 +13370,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS){ @@ -13416,8 +13416,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_P assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13432,6 +13430,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_P TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS){ @@ -13467,8 +13467,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13483,6 +13481,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS){ @@ -13519,8 +13519,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_C assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13535,6 +13533,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_C TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS){ @@ -13589,8 +13589,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARA assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13605,6 +13603,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARA TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS){ @@ -13653,8 +13653,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS){ assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); - start_frame: - TAIL_CALL(start_frame); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -13669,6 +13667,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS){ TAIL_CALL(exception_unwind); exit_unwind: TAIL_CALL(exit_unwind); + start_frame: + TAIL_CALL(start_frame); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS){ diff --git a/Python/generated_tail_call_labels.c.h b/Python/generated_tail_call_labels.c.h index 36ec137f32534f..ef02ea38b87cc2 100644 --- a/Python/generated_tail_call_labels.c.h +++ b/Python/generated_tail_call_labels.c.h @@ -8,30 +8,6 @@ #endif #define TIER_ONE 1 - error: - { - /* Double-check exception status. */ - #ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } - #else - assert(_PyErr_Occurred(tstate)); - #endif - - /* Log traceback info. */ - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); - goto exception_unwind; - } - exception_unwind: { /* We can't use frame->instr_ptr here, as RERAISE may have set it */ @@ -104,4 +80,28 @@ goto error; } + error: + { + /* Double-check exception status. */ + #ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } + #else + assert(_PyErr_Occurred(tstate)); + #endif + + /* Log traceback info. */ + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); + goto exception_unwind; + } + #undef TIER_ONE From a01caf1a2f48f26a0433a41bf8cc4e7ed6e49339 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 1 Feb 2025 21:20:11 +0800 Subject: [PATCH 094/110] Fix --- Python/ceval.c | 4 +- Python/generated_tail_call_labels.c.h | 89 ++++++++++++++----- Tools/cases_generator/tier1_generator.py | 7 +- .../tier1_tail_call_generator.py | 5 +- 4 files changed, 77 insertions(+), 28 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 756d7971d7ff53..4801932603f0c2 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -865,11 +865,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int const _PyUOpInstruction *next_uop = NULL; #endif + goto start_frame; + #ifdef Py_TAIL_CALL_INTERP - return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, 0, 0); # include "generated_tail_call_labels.c.h" #else - goto start_frame; # include "generated_cases.c.h" #endif diff --git a/Python/generated_tail_call_labels.c.h b/Python/generated_tail_call_labels.c.h index ef02ea38b87cc2..32f8ffc7c281c3 100644 --- a/Python/generated_tail_call_labels.c.h +++ b/Python/generated_tail_call_labels.c.h @@ -8,6 +8,54 @@ #endif #define TIER_ONE 1 + pop_4_error: + { + STACK_SHRINK(1); + goto pop_3_error; + } + + pop_3_error: + { + STACK_SHRINK(1); + goto pop_2_error; + } + + pop_2_error: + { + STACK_SHRINK(1); + goto pop_1_error; + } + + pop_1_error: + { + STACK_SHRINK(1); + goto error; + } + + error: + { + /* Double-check exception status. */ + #ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } + #else + assert(_PyErr_Occurred(tstate)); + #endif + + /* Log traceback info. */ + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); + goto exception_unwind; + } + exception_unwind: { /* We can't use frame->instr_ptr here, as RERAISE may have set it */ @@ -55,8 +103,7 @@ lltrace_resume_frame(frame); } #endif - _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); - ; + return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); } exit_unwind: @@ -80,28 +127,30 @@ goto error; } - error: + start_frame: { - /* Double-check exception status. */ - #ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); + if (_Py_EnterRecursivePy(tstate)) { + goto exit_unwind; } - #else - assert(_PyErr_Occurred(tstate)); - #endif - - /* Log traceback info. */ - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + #ifdef LLTRACE + { + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + frame->lltrace = lltrace; + if (lltrace < 0) { + goto exit_unwind; } } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); - goto exception_unwind; + #endif + + #ifdef Py_DEBUG + /* _PyEval_EvalFrameDefault() must not be called with an exception set, + because it can clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!_PyErr_Occurred(tstate)); + #endif + return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); } #undef TIER_ONE diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index e4e48011132ecc..ead11e3bed84a3 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -15,7 +15,6 @@ Flush, analysis_error, StackItem, - Label, ) from generators_common import ( DEFAULT_INPUT, @@ -224,15 +223,15 @@ def generate_tier1( """) out = CWriter(outfile, 2, lines) emitter = Emitter(out) - generate_tier1_labels(analysis.labels, emitter) + generate_tier1_labels(analysis, emitter) outfile.write(f"{LABEL_END_MARKER}\n") outfile.write(FOOTER) def generate_tier1_labels( - labels: dict[str, Label], emitter: Emitter + analysis: Analysis, emitter: Emitter ) -> None: emitter.emit("\n") - for name, label in labels.items(): + for name, label in analysis.labels.items(): emitter.emit(f"{name}:\n") emitter.emit_label(label) emitter.emit("\n") diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index 42dfbd58a16927..42b9981a3bcd4d 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -147,7 +147,8 @@ def dispatch( # Replace DISPATCH with _TAIL_CALL_entry(...) next(tkn_iter) next(tkn_iter) - self.emit("_TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0);\n") + next(tkn_iter) + self.emit("return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0);\n") return True @@ -194,7 +195,7 @@ def generate_tier1( labels_outfile.write(PRELUDE) out = CWriter(labels_outfile, 2, lines) emitter = TailCallCevalLabelsEmitter(out) - generate_tier1_labels({label: analysis.labels[label] for label in NEEDED_LABELS}, emitter) + generate_tier1_labels(analysis, emitter) labels_outfile.write(FOOTER) write_header(__file__, filenames, outfile) outfile.write(PRELUDE) From 26948002ed2db6d762a431ab68de631fb443ff4e Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 1 Feb 2025 21:22:54 +0800 Subject: [PATCH 095/110] Remove unused code --- Tools/cases_generator/tier1_tail_call_generator.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index 42b9981a3bcd4d..2cf58de6faac2a 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -46,12 +46,6 @@ """ FOOTER = "#undef TIER_ONE\n" -NEEDED_LABELS = { - "error", - "exception_unwind", - "exit_unwind", -} - class TailCallEmitter(Emitter): def __init__(self, out: CWriter, analysis: Analysis): From 62a1a4f0a49dacd0f82ef57ec06ee64c7cce8c2d Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 1 Feb 2025 21:58:36 +0800 Subject: [PATCH 096/110] Add tests --- Lib/test/test_generated_cases.py | 123 +++++ Python/generated_tail_call_handlers.c.h | 450 +++++++++--------- .../tier1_tail_call_generator.py | 18 +- 3 files changed, 362 insertions(+), 229 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 5970972d855e75..8f4605443d732b 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1817,6 +1817,129 @@ def test_multiple_labels(self): } """ +class TestGeneratedTailCallHandlers(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.maxDiff = None + + self.temp_dir = tempfile.gettempdir() + self.temp_input_filename = os.path.join(self.temp_dir, "input.txt") + self.temp_output_filename = os.path.join(self.temp_dir, "output.txt") + self.temp_labels_output_filename = os.path.join(self.temp_dir, "labels_output.txt") + + def tearDown(self) -> None: + for filename in [ + self.temp_input_filename, + self.temp_output_filename, + self.temp_labels_output_filename, + ]: + try: + os.remove(filename) + except Exception: + pass + super().tearDown() + + def run_cases_test(self, input: str, expected: str, expected_labels: str): + with open(self.temp_input_filename, "w+") as temp_input: + temp_input.write(parser.BEGIN_MARKER) + temp_input.write(input) + temp_input.write(parser.END_MARKER) + temp_input.flush() + + with handle_stderr(): + tier1_tail_call_generator.generate_tier1_from_files( + [self.temp_input_filename], self.temp_output_filename, self.temp_labels_output_filename, False + ) + + with open(self.temp_output_filename) as temp_output: + lines = temp_output.read() + _, rest = lines.split(tier1_generator.INSTRUCTION_START_MARKER) + instructions, _ = rest.split(tier1_generator.INSTRUCTION_END_MARKER) + + with open(self.temp_labels_output_filename) as temp_output: + lines = temp_output.readlines() + while lines and lines[0].startswith(("// ", "#", " #", "\n")): + lines.pop(0) + while lines and lines[-1].startswith(("#", "\n")): + lines.pop(-1) + actual_labels = "".join(lines) + + self.assertEqual(instructions.strip(), expected.strip()) + self.assertEqual(actual_labels.strip(), expected_labels.strip()) + + def test_basic(self): + input = """ + inst(OP, (--)) { + SPAM(); + } + """ + output = """ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_OP(TAIL_CALL_PARAMS) { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + SPAM(); + } + DISPATCH(); +} + """ + output_labels = "" + self.run_cases_test(input, output, output_labels) + + def test_label_transformed(self): + """This tests that the labels and their gotos/DISPATCH get transformed to tail calls in the + tail-calling interpreter, while staying the same/becoming an entry call in the normal interpreter. + """ + input = """ + inst(OP, (--)) { + SPAM(); + } + + label(hello) { + EGGS(); + if (x) { + goto baz; + } + DISPATCH(); + } + """ + output = """ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_hello(TAIL_CALL_PARAMS) +{ + EGGS(); + if (x) { + TAIL_CALL(baz); + } + DISPATCH(); +} + + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_OP(TAIL_CALL_PARAMS) { + { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + SPAM(); + } + DISPATCH(); + hello: + TAIL_CALL(hello); +} + """ + output_labels = """ +hello: + { + EGGS(); + if (x) { + goto baz; + } + return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); + } + """ + self.run_cases_test(input, output, output_labels) + + class TestGeneratedAbstractCases(unittest.TestCase): def setUp(self) -> None: super().setUp() diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index bf3aa9ccda127e..30ffef94a6b53a 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -19,6 +19,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); +/* BEGIN INSTRUCTIONS */ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_4_error(TAIL_CALL_PARAMS) { STACK_SHRINK(1); @@ -166,8 +167,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS) } - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 6; @@ -235,7 +235,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -300,7 +300,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -364,7 +364,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -428,7 +428,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -497,7 +497,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAM TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -591,7 +591,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -656,7 +656,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -720,7 +720,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -785,7 +785,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -849,7 +849,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -917,7 +917,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 2; @@ -982,7 +982,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -1040,7 +1040,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PAR TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -1138,7 +1138,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -1217,7 +1217,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -1288,7 +1288,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -1355,7 +1355,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CAL TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -1391,7 +1391,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -1449,7 +1449,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -1513,7 +1513,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -1557,7 +1557,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -1610,7 +1610,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -1646,7 +1646,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -1673,7 +1673,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 4; @@ -1852,7 +1852,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -1989,7 +1989,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_C TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -2143,7 +2143,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -2281,7 +2281,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_C TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -2386,7 +2386,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -2497,7 +2497,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -2609,7 +2609,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS( TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -2712,7 +2712,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS) TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -2893,7 +2893,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -2929,7 +2929,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAM TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -2972,7 +2972,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAM TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -3046,7 +3046,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 4; @@ -3222,7 +3222,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -3367,7 +3367,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_P TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -3488,7 +3488,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -3604,7 +3604,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -3678,7 +3678,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -3747,7 +3747,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAM TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -3864,7 +3864,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -3981,7 +3981,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -4092,7 +4092,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -4206,7 +4206,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -4321,7 +4321,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -4445,7 +4445,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PAR TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -4554,7 +4554,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -4632,7 +4632,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -4710,7 +4710,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -4765,7 +4765,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -4830,7 +4830,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS) TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -4876,7 +4876,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -4936,7 +4936,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 2; @@ -5015,7 +5015,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -5080,7 +5080,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAM TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -5157,7 +5157,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS) TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -5223,7 +5223,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS) TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 2; @@ -5287,7 +5287,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -5338,7 +5338,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAM TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -5390,7 +5390,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -5428,7 +5428,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -5461,7 +5461,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -5497,7 +5497,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS) TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -5532,7 +5532,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -5570,7 +5570,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -5606,7 +5606,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -5646,7 +5646,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -5693,7 +5693,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -5732,7 +5732,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -5779,7 +5779,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -5830,7 +5830,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -5879,7 +5879,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS) { { next_instr += 1; INSTRUCTION_STATS(END_FOR); @@ -5913,7 +5913,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -5949,7 +5949,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -6000,7 +6000,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS) TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -6038,7 +6038,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -6067,7 +6067,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -6110,7 +6110,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -6150,7 +6150,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAM TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 2; @@ -6230,7 +6230,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -6309,7 +6309,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -6383,7 +6383,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -6449,7 +6449,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS) TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -6520,7 +6520,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS) TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -6585,7 +6585,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -6623,7 +6623,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -6658,7 +6658,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -6694,7 +6694,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -6733,7 +6733,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -6794,7 +6794,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -6831,7 +6831,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -6874,7 +6874,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -7061,7 +7061,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -7087,7 +7087,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -7129,7 +7129,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_P TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -7172,7 +7172,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_P TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -7217,7 +7217,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -7272,7 +7272,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -7309,7 +7309,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -7351,7 +7351,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -7378,7 +7378,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_C TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -7434,7 +7434,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -7464,7 +7464,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAI TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -7493,7 +7493,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -7526,7 +7526,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -7560,7 +7560,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(T TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -7596,7 +7596,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -7630,7 +7630,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NON TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -7664,7 +7664,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -7754,7 +7754,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -7815,7 +7815,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_C TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -7900,7 +7900,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -7937,7 +7937,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAM TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -7974,7 +7974,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 2; @@ -8035,7 +8035,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -8117,7 +8117,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -8149,7 +8149,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 2; @@ -8197,7 +8197,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_P TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -8223,7 +8223,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -8257,7 +8257,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -8312,7 +8312,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 10; @@ -8406,7 +8406,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -8471,7 +8471,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -8546,7 +8546,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_C TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -8617,7 +8617,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDE TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -8705,7 +8705,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -8775,7 +8775,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -8834,7 +8834,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -8915,7 +8915,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAI TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -9005,7 +9005,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -9059,7 +9059,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT( TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -9134,7 +9134,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VAL TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -9242,7 +9242,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -9318,7 +9318,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -9435,7 +9435,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -9477,7 +9477,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAM TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -9517,7 +9517,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_P TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -9568,7 +9568,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -9601,7 +9601,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_PA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_MORTAL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_MORTAL(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -9633,7 +9633,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_MORTAL(TAIL_CALL_PARA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -9671,7 +9671,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -9702,7 +9702,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -9734,7 +9734,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -9774,7 +9774,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -9809,7 +9809,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -9862,7 +9862,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CAL TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -9950,7 +9950,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_C TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 5; @@ -10016,7 +10016,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -10113,7 +10113,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -10194,7 +10194,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PAR TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -10232,7 +10232,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -10267,7 +10267,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -10299,7 +10299,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS) TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -10354,7 +10354,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 2; @@ -10474,7 +10474,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -10536,7 +10536,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_P TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -10610,7 +10610,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -10643,7 +10643,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -10682,7 +10682,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -10727,7 +10727,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -10783,7 +10783,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -10823,7 +10823,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -10856,7 +10856,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -10889,7 +10889,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS) TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -10914,7 +10914,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -10939,7 +10939,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -10974,7 +10974,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -11004,7 +11004,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -11039,7 +11039,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -11090,7 +11090,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAM TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -11141,7 +11141,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_P TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -11176,7 +11176,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAM TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -11206,7 +11206,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -11249,7 +11249,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -11279,7 +11279,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -11322,7 +11322,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -11383,7 +11383,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -11410,7 +11410,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -11498,7 +11498,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -11549,7 +11549,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -11600,7 +11600,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAM TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -11646,7 +11646,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 2; @@ -11754,7 +11754,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -11837,7 +11837,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -11892,7 +11892,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -11929,7 +11929,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -11971,7 +11971,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -12008,7 +12008,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 5; @@ -12071,7 +12071,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -12160,7 +12160,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_C TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -12224,7 +12224,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -12337,7 +12337,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -12370,7 +12370,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -12400,7 +12400,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -12433,7 +12433,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_P TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -12468,7 +12468,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -12503,7 +12503,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -12555,7 +12555,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -12621,7 +12621,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 2; @@ -12684,7 +12684,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -12734,7 +12734,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -12813,7 +12813,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -12849,7 +12849,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 4; @@ -12907,7 +12907,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -12957,7 +12957,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -12994,7 +12994,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -13042,7 +13042,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -13084,7 +13084,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -13125,7 +13125,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -13174,7 +13174,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -13209,7 +13209,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -13244,7 +13244,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS) TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -13276,7 +13276,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -13313,7 +13313,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS){ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 2; @@ -13374,7 +13374,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -13434,7 +13434,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_P TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -13485,7 +13485,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_ TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS) { { _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -13537,7 +13537,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_C TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -13607,7 +13607,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARA TAIL_CALL(start_frame); } -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS){ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS) { { frame->instr_ptr = next_instr; next_instr += 1; @@ -13670,8 +13670,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS){ start_frame: TAIL_CALL(start_frame); } - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS){ +/* END INSTRUCTIONS */ +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { { _PyErr_Format(tstate, PyExc_SystemError, diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index 2cf58de6faac2a..8bd66e76edadb5 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -28,6 +28,8 @@ from tier1_generator import ( write_single_inst, generate_tier1_labels, + INSTRUCTION_START_MARKER, + INSTRUCTION_END_MARKER, ) from lexer import Token @@ -142,6 +144,7 @@ def dispatch( next(tkn_iter) next(tkn_iter) next(tkn_iter) + self.out.start_line() self.emit("return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0);\n") return True @@ -158,6 +161,7 @@ def generate_label_handlers( for name in analysis.labels: emitter.emit(f"{function_proto(name)};\n") emitter.emit("\n") + out.emit(f"{INSTRUCTION_START_MARKER}\n") for name, label in analysis.labels.items(): emitter.emit(f"{function_proto(name)}\n") emitter.emit_label(label) @@ -200,11 +204,10 @@ def generate_tier1( generate_label_handlers(analysis, outfile, lines) emitter = TailCallEmitter(out, analysis) - out.emit("\n") for name, inst in sorted(analysis.instructions.items()): out.emit("\n") out.emit(function_proto(name)) - out.emit("{\n") + out.emit(" {\n") # We wrap this with a block to signal to GCC that the local variables # are dead at the tail call site. # Otherwise, GCC 15's escape analysis may think there are @@ -225,11 +228,11 @@ def generate_tier1( out.start_line() out.emit("}\n") - out.emit("\n") + out.emit(f"{INSTRUCTION_END_MARKER}\n") # Emit unknown opcode handler. out.emit(function_proto("UNKNOWN_OPCODE")) - out.emit("{\n") + out.emit(" {\n") out.emit("{\n") out.emit(""" _PyErr_Format(tstate, PyExc_SystemError, @@ -252,6 +255,13 @@ def generate_tier1( out.emit("};\n") outfile.write(FOOTER) +def generate_tier1_from_files( + filenames: list[str], out_filename: str, out_label_filename: str, lines: bool +) -> None: + data = analyze_files(filenames) + with open(out_filename, "w") as outfile, open(out_label_filename, "w") as labels_outfile: + generate_tier1(filenames, data, outfile, labels_outfile, lines) + arg_parser = argparse.ArgumentParser( description="Generate the code for the interpreter switch.", From da2b792392414c13381e13a2275b4ca2d5665da6 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 1 Feb 2025 23:29:57 +0800 Subject: [PATCH 097/110] fix configure description --- configure | 3 +-- configure.ac | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/configure b/configure index bee07338166d77..22456959add174 100755 --- a/configure +++ b/configure @@ -1931,8 +1931,7 @@ Optional Packages: --with-computed-gotos enable computed gotos in evaluation loop (enabled by default on supported compilers) --tail-call-interp enable tail-calling interpreter in evaluation loop - and rest of CPython (enabled by default on supported - compilers) + and rest of CPython --with-ensurepip[=install|upgrade|no] "install" or "upgrade" using bundled pip (default is upgrade) diff --git a/configure.ac b/configure.ac index 2440291b7ea55e..cf16e77f0a1503 100644 --- a/configure.ac +++ b/configure.ac @@ -7023,7 +7023,7 @@ AC_ARG_WITH( [tail-call-interp], [AS_HELP_STRING( [--tail-call-interp], - [enable tail-calling interpreter in evaluation loop and rest of CPython (enabled by default on supported compilers)] + [enable tail-calling interpreter in evaluation loop and rest of CPython] )], [ if test "$withval" = yes From 85d5fc3785c54126458b83b394b4d328ed7b0ac8 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 3 Feb 2025 23:23:46 +0800 Subject: [PATCH 098/110] regen changes from upstream --- Python/generated_tail_call_handlers.c.h | 1343 +++++++++++++++++------ Python/generated_tail_call_labels.c.h | 12 +- 2 files changed, 1037 insertions(+), 318 deletions(-) diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 30ffef94a6b53a..78e5d20115acfb 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -22,20 +22,20 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); /* BEGIN INSTRUCTIONS */ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_4_error(TAIL_CALL_PARAMS) { - STACK_SHRINK(1); - TAIL_CALL(pop_3_error); + STACK_SHRINK(4); + TAIL_CALL(error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_3_error(TAIL_CALL_PARAMS) { - STACK_SHRINK(1); - TAIL_CALL(pop_2_error); + STACK_SHRINK(3); + TAIL_CALL(error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS) { - STACK_SHRINK(1); - TAIL_CALL(pop_1_error); + STACK_SHRINK(2); + TAIL_CALL(error); } Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS) @@ -209,7 +209,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS) { stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(lhs); PyStackRef_CLOSE(rhs); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + goto pop_2_error; + } res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -274,7 +276,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PA ((PyFloatObject *)left_o)->ob_fval + ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + goto pop_2_error; + } res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -338,7 +342,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARA PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + goto pop_2_error; + } res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -402,7 +408,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_ PyObject *res_o = PyUnicode_Concat(left_o, right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + goto pop_2_error; + } res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -561,7 +569,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TA PyUnicode_Append(&temp, right_o); *target_local = PyStackRef_FromPyObjectSteal(temp); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (PyStackRef_IsNull(*target_local)) goto pop_2_error; + if (PyStackRef_IsNull(*target_local)) { + goto pop_2_error; + } #if TIER_ONE // The STORE_FAST is already done. This is done here in tier one, // and during trace projection in tier two: @@ -630,7 +640,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CA ((PyFloatObject *)left_o)->ob_fval * ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + goto pop_2_error; + } res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -694,7 +706,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + goto pop_2_error; + } res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -759,7 +773,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CA ((PyFloatObject *)left_o)->ob_fval - ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + goto pop_2_error; + } res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -823,7 +839,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + goto pop_2_error; + } res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -890,12 +908,18 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS) { stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); } + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(container); - if (res_o == NULL) goto pop_3_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[-3] = res; - stack_pointer += -2; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -956,7 +980,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS) stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + goto pop_2_error; + } res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; @@ -1014,7 +1040,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PAR } PyStackRef_CLOSE(dict_st); PyStackRef_CLOSE(sub_st); - if (rc <= 0) goto pop_2_error; + if (rc <= 0) { + goto pop_2_error; + } // not found or error res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1192,10 +1220,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL Py_INCREF(res_o); #endif PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(list_st); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -1263,10 +1295,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_ STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(str_st); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -1330,10 +1366,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CAL assert(res_o != NULL); Py_INCREF(res_o); PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(tuple_st); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -1404,11 +1444,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS) { for (int _i = oparg*2; --_i >= 0;) { PyStackRef_CLOSE(values[_i]); } - { - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *map_o = _PyDict_FromItems( @@ -1464,11 +1502,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS) { for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(values[_i]); } - { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } int err = 0; for (int i = 0; i < oparg; i++) { @@ -1570,11 +1606,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS) { for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(pieces[_i]); } - { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); @@ -1681,6 +1715,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS) { PREDICTED_CALL:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; + opcode = CALL; _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; @@ -1718,7 +1753,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS) { PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable[0]; func[0] = PyStackRef_FromPyObjectNew(method); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } } // _DO_CALL @@ -1766,11 +1803,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS) { for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( @@ -1824,7 +1859,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS) { _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -1921,7 +1958,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_C self[0] = PyStackRef_FromPyObjectSteal(self_o); _PyStackRef temp = callable[0]; init[0] = PyStackRef_FromPyObjectNew(init_func); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } // _CREATE_INIT_FRAME { @@ -1998,8 +2037,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; _PyStackRef *null; - _PyStackRef *func; - _PyStackRef *self; _PyStackRef *self_or_null; _PyStackRef *args; _PyInterpreterFrame *new_frame; @@ -2029,19 +2066,20 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { - func = &stack_pointer[-2 - oparg]; - self = &stack_pointer[-1 - oparg]; + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null[0])); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); STAT_INC(CALL, hit); - self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + self_or_null[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + callable[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } // flush // _CHECK_FUNCTION_VERSION { - callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); if (!PyFunction_Check(callable_o)) { @@ -2058,7 +2096,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI } // _CHECK_FUNCTION_EXACT_ARGS { - self_or_null = &stack_pointer[-1 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; @@ -2152,8 +2189,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_C static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; _PyStackRef *null; - _PyStackRef *method; - _PyStackRef *self; _PyStackRef *self_or_null; _PyStackRef *args; _PyInterpreterFrame *new_frame; @@ -2196,23 +2231,22 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_C } // _EXPAND_METHOD { - method = &stack_pointer[-2 - oparg]; - self = &stack_pointer[-1 - oparg]; + self_or_null = null; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - assert(PyStackRef_IsNull(null[0])); + assert(PyStackRef_IsNull(self_or_null[0])); assert(Py_TYPE(callable_o) == &PyMethod_Type); - self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + self_or_null[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); _PyStackRef temp = callable[0]; - method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(method[0])); + callable[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable[0])); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } // flush // _PY_FRAME_GENERAL { args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; @@ -2325,11 +2359,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); @@ -2358,7 +2390,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -2432,11 +2466,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARA for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( @@ -2469,7 +2501,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARA _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -2547,11 +2581,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS( for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); @@ -2581,7 +2613,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS( _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -2664,11 +2698,15 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS) stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable[0]); + stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); goto error; } res = PyStackRef_FromPyObjectSteal(res_o); @@ -2678,19 +2716,21 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS) _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 1 + oparg; + if (err != 0) { + goto error; + } + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -2714,18 +2754,18 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS) { { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(CALL_FUNCTION_EX); - PREDICTED_CALL_FUNCTION_EX:; - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; + opcode = CALL_FUNCTION_EX; _PyStackRef func; _PyStackRef callargs; _PyStackRef kwargs_in; _PyStackRef tuple; _PyStackRef kwargs_out; _PyStackRef func_st; + _PyStackRef null; _PyStackRef callargs_st; _PyStackRef kwargs_st; _PyStackRef result; @@ -2753,15 +2793,23 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM goto error; } kwargs_out = kwargs_in; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callargs); + stack_pointer = _PyFrame_GetStackPointer(frame); tuple = PyStackRef_FromPyObjectSteal(tuple_o); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } } // _DO_CALL_FUNCTION_EX { kwargs_st = kwargs_out; callargs_st = tuple; + null = stack_pointer[-3]; func_st = func; + (void)null; PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. @@ -2819,7 +2867,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -3; + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( @@ -2827,7 +2875,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM nargs, callargs, kwargs, frame); stack_pointer = _PyFrame_GetStackPointer(frame); // Need to sync the stack since we exit with DISPATCH_INLINED. - stack_pointer += -1; + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { goto error; @@ -2846,12 +2894,24 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM result_o = PyObject_Call(func, callargs, kwargs); stack_pointer = _PyFrame_GetStackPointer(frame); } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(kwargs_st); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(func_st); - if (result_o == NULL) goto pop_4_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + goto error; + } result = PyStackRef_FromPyObjectSteal(result_o); } // _CHECK_PERIODIC @@ -2859,19 +2919,21 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-4] = result; - stack_pointer += -3; + stack_pointer[0] = result; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 3; + if (err != 0) { + goto error; + } + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-4] = result; - stack_pointer += -3; + stack_pointer[0] = result; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -2906,7 +2968,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAM PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (res_o == NULL) goto pop_1_error; + if (res_o == NULL) { + goto pop_1_error; + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; } @@ -2947,7 +3011,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAM stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value2_st); PyStackRef_CLOSE(value1_st); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + goto pop_2_error; + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -3054,6 +3120,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS) { PREDICTED_CALL_KW:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; + opcode = CALL_KW; _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; @@ -3095,7 +3162,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS) { PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable[0]; func[0] = PyStackRef_FromPyObjectNew(method); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } kwnames_out = kwnames_in; } @@ -3129,9 +3198,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS) { arguments, positional_args, kwnames_o, frame ); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); // Sync stack explicitly since we leave using DISPATCH_INLINED(). - stack_pointer += -3 - oparg; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. @@ -3151,11 +3224,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS) { PyStackRef_CLOSE(args[_i]); } PyStackRef_CLOSE(kwnames); - { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } stack_pointer[-1] = kwnames; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3232,8 +3303,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_P _PyStackRef *callable; _PyStackRef *null; _PyStackRef kwnames; - _PyStackRef *method; - _PyStackRef *self; _PyStackRef *self_or_null; _PyStackRef *args; _PyInterpreterFrame *new_frame; @@ -3276,24 +3345,23 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_P } // _EXPAND_METHOD_KW { - method = &stack_pointer[-3 - oparg]; - self = &stack_pointer[-2 - oparg]; + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null[0])); _PyStackRef callable_s = callable[0]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable_s); - assert(PyStackRef_IsNull(null[0])); assert(Py_TYPE(callable_o) == &PyMethod_Type); - self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(method[0])); + self_or_null[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + callable[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable[0])); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable_s); + stack_pointer = _PyFrame_GetStackPointer(frame); } // flush // _PY_FRAME_KW { kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; @@ -3313,10 +3381,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_P arguments, positional_args, kwnames_o, frame ); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. - stack_pointer += -3 - oparg; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { goto error; @@ -3373,6 +3445,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_NON_PY); + opcode = CALL_KW_NON_PY; static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef *callable; _PyStackRef kwnames; @@ -3420,11 +3493,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) PyStackRef_CLOSE(args[_i]); } PyStackRef_CLOSE(kwnames); - { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); @@ -3434,7 +3505,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames_o); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(callable[0]); @@ -3443,7 +3518,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) PyStackRef_CLOSE(args[_i]); } if (res_o == NULL) { - stack_pointer += -3 - oparg; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); goto error; } @@ -3454,19 +3529,21 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 2 + oparg; + if (err != 0) { + goto error; + } + stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -3550,10 +3627,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS) { arguments, positional_args, kwnames_o, frame ); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. - stack_pointer += -3 - oparg; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { goto error; @@ -3652,11 +3733,17 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS) { if (res_o == NULL) { GOTO_ERROR(error); } + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg_stackref); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable[0]); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -3716,17 +3803,25 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAM STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(self); - PyStackRef_CLOSE(callable); - if (err) goto pop_3_error; - #if TIER_ONE - // Skip the following POP_TOP. This is done here in tier one, and + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } + #if TIER_ONE + // Skip the following POP_TOP. This is done here in tier one, and // during trace projection in tier two: assert(next_instr->op.code == POP_TOP); SKIP_OVER(1); #endif - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); pop_4_error: @@ -3800,11 +3895,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFast cfunc = @@ -3836,7 +3929,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -3917,11 +4012,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFastWithKeywords cfunc = @@ -3953,7 +4046,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -4044,11 +4139,15 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TA stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(self_stackref); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable[0]); + stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); goto error; } res = PyStackRef_FromPyObjectSteal(res_o); @@ -4058,19 +4157,21 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TA _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 1 + oparg; + if (err != 0) { + goto error; + } + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -4112,8 +4213,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CA callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; + _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { - args--; + arguments--; total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; @@ -4139,8 +4241,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CA assert(_PyOpcode_Deopt[opcode] == (CALL)); Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); } - _PyStackRef arg_stackref = args[1]; - _PyStackRef self_stackref = args[0]; + _PyStackRef arg_stackref = arguments[1]; + _PyStackRef self_stackref = arguments[0]; if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); @@ -4157,9 +4259,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CA stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(self_stackref); - PyStackRef_CLOSE(arg_stackref); PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -4178,7 +4282,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CA _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -4212,6 +4318,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PA (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_NON_PY_GENERAL); + opcode = CALL_NON_PY_GENERAL; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; _PyStackRef *self_or_null; @@ -4256,11 +4363,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PA for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( @@ -4293,7 +4398,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PA _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -4589,8 +4696,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS) { _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Str(arg_o); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); - if (res_o == NULL) goto pop_3_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC @@ -4598,19 +4711,21 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS) { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-3] = res; - stack_pointer += -2; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 2; + if (err != 0) { + goto error; + } + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-3] = res; - stack_pointer += -2; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -4667,8 +4782,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS) { _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PySequence_Tuple(arg_o); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); - if (res_o == NULL) goto pop_3_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC @@ -4676,19 +4797,21 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS) { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-3] = res; - stack_pointer += -2; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 2; + if (err != 0) { + goto error; + } + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-3] = res; - stack_pointer += -2; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); @@ -4741,10 +4864,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS) { } STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); - PyStackRef_CLOSE(arg); stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); } DISPATCH(); pop_4_error: @@ -4794,9 +4919,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS) stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); - if (res < 0) goto pop_2_error; + if (res < 0) { + goto pop_2_error; + } assert((match_o == NULL) == (rest_o == NULL)); - if (match_o == NULL) goto pop_2_error; + if (match_o == NULL) { + goto pop_2_error; + } if (!Py_IsNone(match_o)) { stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -4975,7 +5104,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS) { stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + goto pop_2_error; + } if (oparg & 16) { stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -4983,7 +5114,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS) { int res_bool = PyObject_IsTrue(res_o); Py_DECREF(res_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_bool < 0) goto error; + if (res_bool < 0) { + goto error; + } res = res_bool ? PyStackRef_True : PyStackRef_False; } else { @@ -5261,7 +5394,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS) { stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); - if (res < 0) goto pop_2_error; + if (res < 0) { + goto pop_2_error; + } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } stack_pointer[-2] = b; @@ -5313,7 +5448,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAM stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); - if (res < 0) goto pop_2_error; + if (res < 0) { + goto pop_2_error; + } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -5365,7 +5502,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); - if (res < 0) goto pop_2_error; + if (res < 0) { + goto pop_2_error; + } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -5404,10 +5543,18 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS) _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); - if (result_o == NULL) goto pop_1_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + goto error; + } result = PyStackRef_FromPyObjectSteal(result_o); - stack_pointer[-1] = result; + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); pop_4_error: @@ -5509,7 +5656,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS) { int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); - if (err) goto pop_1_error; + if (err) { + goto pop_1_error; + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } @@ -5709,7 +5858,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS) stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); - if (err) goto pop_2_error; + if (err) { + goto pop_2_error; + } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); } @@ -5890,9 +6041,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS) { * This has the benign side effect that if value is * finalized it will see the location as the FOR_ITER's. */ - PyStackRef_CLOSE(value); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); } DISPATCH(); pop_4_error: @@ -5955,6 +6108,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS) (void)this_instr; next_instr += 1; INSTRUCTION_STATS(ENTER_EXECUTOR); + opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; @@ -6043,6 +6197,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXTENDED_ARG); + opcode = EXTENDED_ARG; assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; @@ -6082,14 +6237,24 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS) _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Format(value_o, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); - if (res_o == NULL) goto pop_1_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); } else { res = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); pop_4_error: @@ -6125,7 +6290,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAM stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); PyStackRef_CLOSE(fmt_spec); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + goto pop_2_error; + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -6423,7 +6590,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS) r->start = value + r->step; r->len--; PyObject *res = PyLong_FromLong(value); - if (res == NULL) goto error; + if (res == NULL) { + goto error; + } next = PyStackRef_FromPyObjectSteal(res); } stack_pointer[0] = next; @@ -6549,7 +6718,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS) { iter_o = (*getter)(obj_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); - if (iter_o == NULL) goto pop_1_error; + if (iter_o == NULL) { + goto pop_1_error; + } if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { stack_pointer += -1; @@ -6635,7 +6806,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS) PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); - if (iter_o == NULL) goto pop_1_error; + if (iter_o == NULL) { + goto pop_1_error; + } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; } @@ -6671,7 +6844,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS) { PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); - if (iter_o == NULL) goto pop_1_error; + if (iter_o == NULL) { + goto pop_1_error; + } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; } @@ -6706,9 +6881,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS) { _PyFrame_SetStackPointer(frame, stack_pointer); Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) goto error; + if (len_i < 0) { + goto error; + } PyObject *len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) goto error; + if (len_o == NULL) { + goto error; + } len = PyStackRef_FromPyObjectSteal(len_o); stack_pointer[0] = len; stack_pointer += 1; @@ -6806,7 +6985,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS) { _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) goto error; + if (res_o == NULL) { + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; @@ -6849,7 +7030,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS) { stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(level); PyStackRef_CLOSE(fromlist); - if (res_o == NULL) goto pop_2_error; + if (res_o == NULL) { + goto pop_2_error; + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; @@ -6880,6 +7063,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARA (void)this_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL); + opcode = INSTRUMENTED_CALL; _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; @@ -6901,7 +7085,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARA PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable[0]; func[0] = PyStackRef_FromPyObjectNew(method); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } } // _MONITOR_CALL @@ -6929,7 +7115,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARA frame, this_instr, function, arg0 ); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; + if (err) { + goto error; + } } // _DO_CALL { @@ -6975,11 +7163,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARA for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( @@ -7033,7 +7219,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARA _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -7067,8 +7255,185 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TA (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - Py_MUSTTAIL return (INSTRUCTION_TABLE[CALL_FUNCTION_EX])(frame, stack_pointer, tstate, this_instr, opcode, oparg); + opcode = INSTRUMENTED_CALL_FUNCTION_EX; + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef kwargs_in; + _PyStackRef tuple; + _PyStackRef kwargs_out; + _PyStackRef func_st; + _PyStackRef null; + _PyStackRef callargs_st; + _PyStackRef kwargs_st; + _PyStackRef result; + // _MAKE_CALLARGS_A_TUPLE + { + kwargs_in = stack_pointer[-1]; + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (PyTuple_CheckExact(callargs_o)) { + tuple = callargs; + kwargs_out = kwargs_in; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { + goto error; + } + kwargs_out = kwargs_in; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + tuple = PyStackRef_FromPyObjectSteal(tuple_o); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } + } + // _DO_CALL_FUNCTION_EX + { + kwargs_st = kwargs_out; + callargs_st = tuple; + null = stack_pointer[-3]; + func_st = func; + (void)null; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-2] = callargs_st; + stack_pointer[-1] = kwargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(result_o); + } + } + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Need to sync the stack since we exit with DISPATCH_INLINED. + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + goto error; + } + assert( 1 == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + stack_pointer[-2] = callargs_st; + stack_pointer[-1] = kwargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(kwargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(func_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + goto error; + } + result = PyStackRef_FromPyObjectSteal(result_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + goto error; + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } + DISPATCH(); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7093,24 +7458,171 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_P (void)this_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - uint32_t version = read_u32(&this_instr[2].cache); - (void)version; - int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2)); - int total_args = oparg + is_meth; - PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3)); - PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING - : PyStackRef_AsPyObjectBorrow(PEEK(total_args + 1)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - Py_MUSTTAIL return (INSTRUCTION_TABLE[CALL_KW])(frame, stack_pointer, tstate, this_instr, opcode, oparg); + opcode = INSTRUMENTED_CALL_KW; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef kwnames_in; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef kwnames_out; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _MONITOR_CALL_KW + { + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + int is_meth = !PyStackRef_IsNull(self_or_null[0]); + PyObject *arg; + if (is_meth) { + arg = PyStackRef_AsPyObjectBorrow(self_or_null[0]); + } + else { + if (args) { + arg = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg = &_PyInstrumentation_MISSING; + } + } + PyObject *function = PyStackRef_AsPyObjectBorrow(callable[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } + } + // _MAYBE_EXPAND_METHOD_KW + { + kwnames_in = stack_pointer[-1]; + func = &stack_pointer[-3 - oparg]; + maybe_self = &stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + kwnames_out = kwnames_in; + } + // _DO_CALL_KW + { + kwnames = kwnames_out; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-1] = kwnames; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Sync stack explicitly since we leave using DISPATCH_INLINED(). + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; + } + assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4 ; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + stack_pointer[-1] = kwnames; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(res_o); + } + } + } + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + if (res_o == NULL) { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); } + DISPATCH(); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7193,10 +7705,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_ } } val = value; - PyStackRef_CLOSE(receiver); stack_pointer[-2] = val; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(receiver); + stack_pointer = _PyFrame_GetStackPointer(frame); } DISPATCH(); pop_4_error: @@ -7278,11 +7792,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CA (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + opcode = INSTRUMENTED_INSTRUCTION; _PyFrame_SetStackPointer(frame, stack_pointer); int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_opcode < 0) goto error; + if (next_opcode < 0) { + goto error; + } next_instr = this_instr; if (_PyOpcode_Caches[next_opcode]) { PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); @@ -7324,7 +7841,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } } } // _MONITOR_JUMP_BACKWARD @@ -7385,6 +7904,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARA (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_LINE); + opcode = INSTRUMENTED_LINE; int original_opcode = 0; if (tstate->tracing) { PyCodeObject *code = _PyFrame_GetCode(frame); @@ -7440,12 +7960,89 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAI (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); + opcode = INSTRUMENTED_LOAD_SUPER_ATTR; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ - // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we - // don't want to specialize instrumented instructions - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - Py_MUSTTAIL return (INSTRUCTION_TABLE[LOAD_SUPER_ATTR])(frame, stack_pointer, tstate, this_instr, opcode, oparg); + // _LOAD_SUPER_ATTR + { + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + goto pop_3_error; + } + } + // we make no attempt to optimize here; specializations should + // handle any case whose performance we care about + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(super); + } + } + } + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + if (super == NULL) { + goto pop_3_error; + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + Py_DECREF(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + goto error; + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + } + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } + stack_pointer[0] = attr; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); } + DISPATCH(); pop_4_error: TAIL_CALL(pop_4_error); pop_3_error: @@ -7503,9 +8100,11 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_ _PyStackRef iter; iter = stack_pointer[-1]; INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); - PyStackRef_CLOSE(iter); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); } DISPATCH(); pop_4_error: @@ -7574,7 +8173,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TA INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } else { + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value_stackref); + stack_pointer = _PyFrame_GetStackPointer(frame); } } DISPATCH(); @@ -7607,7 +8208,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NON int jump = !PyStackRef_IsNone(value_stackref); RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); if (jump) { + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value_stackref); + stack_pointer = _PyFrame_GetStackPointer(frame); INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } } @@ -7679,7 +8282,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PA _Py_CODEUNIT *bytecode = _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) goto error; + if (bytecode == NULL) { + goto error; + } _PyFrame_SetStackPointer(frame, stack_pointer); ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7718,7 +8323,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PA _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } } } } @@ -7728,7 +8335,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PA int err = _Py_call_instrumentation( tstate, oparg > 0, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; + if (err) { + goto error; + } if (frame->instr_ptr != this_instr) { /* Instrumentation has jumped */ next_instr = frame->instr_ptr; @@ -7771,7 +8380,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_C tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; + if (err) { + goto error; + } } // _RETURN_VALUE { @@ -8002,7 +8613,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS) _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } } } // _JUMP_BACKWARD_NO_INTERRUPT @@ -8051,7 +8664,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARA _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } } } // _JUMP_BACKWARD_NO_INTERRUPT @@ -8081,7 +8696,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARA stack_pointer = _PyFrame_GetStackPointer(frame); if (optimized <= 0) { this_instr[1].counter = restart_backoff_counter(counter); - if (optimized < 0) goto error; + if (optimized < 0) { + goto error; + } } else { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -8164,7 +8781,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_P _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } } } // _JUMP_BACKWARD_NO_INTERRUPT @@ -8234,7 +8853,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS) { list = stack_pointer[-2 - (oparg-1)]; int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), PyStackRef_AsPyObjectSteal(v)); - if (err < 0) goto pop_1_error; + if (err < 0) { + goto pop_1_error; + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } @@ -8369,7 +8990,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS) { meth | NULL | arg1 | ... | argN */ PyStackRef_CLOSE(owner); - if (attr_o == NULL) goto pop_1_error; + if (attr_o == NULL) { + goto pop_1_error; + } self_or_null[0] = PyStackRef_NULL; } } @@ -8379,7 +9002,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS) { attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); - if (attr_o == NULL) goto pop_1_error; + if (attr_o == NULL) { + goto pop_1_error; + } } attr = PyStackRef_FromPyObjectSteal(attr_o); } @@ -8674,14 +9299,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL { null = PyStackRef_NULL; } - stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -8974,14 +9601,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM attr = PyStackRef_FromPyObjectSteal(attr_o); #endif STAT_INC(LOAD_ATTR, hit); + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL { null = PyStackRef_NULL; } - stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -9445,7 +10074,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAM _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) goto error; + if (err < 0) { + goto error; + } if (bc_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_NameError, @@ -9839,9 +10470,15 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CAL goto error; } } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(class_dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectSteal(value_o); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); pop_4_error: @@ -9876,7 +10513,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_C int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(mod_or_class_dict); - if (err < 0) goto pop_1_error; + if (err < 0) { + goto pop_1_error; + } if (v_o == NULL) { if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) @@ -9908,13 +10547,17 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_C _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) goto error; + if (err < 0) { + goto error; + } if (v_o == NULL) { /* namespace 2: builtins */ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) goto error; + if (err < 0) { + goto error; + } if (v_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg( @@ -9987,7 +10630,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*res)) goto error; + if (PyStackRef_IsNull(*res)) { + goto error; + } } // _PUSH_NULL_CONDITIONAL { @@ -10242,7 +10887,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS) { _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *v_o = _PyEval_LoadName(tstate, frame, name); stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) goto error; + if (v_o == NULL) { + goto error; + } v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; @@ -10362,6 +11009,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS PREDICTED_LOAD_SUPER_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + opcode = LOAD_SUPER_ATTR; _PyStackRef global_super_st; _PyStackRef class_st; _PyStackRef self_st; @@ -10435,7 +11083,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); - if (super == NULL) goto pop_3_error; + if (super == NULL) { + goto pop_3_error; + } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); @@ -10443,7 +11093,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS PyObject *attr_o = PyObject_GetAttr(super, name); Py_DECREF(super); stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) goto error; + if (attr_o == NULL) { + goto error; + } attr = PyStackRef_FromPyObjectSteal(attr_o); } // _PUSH_NULL_CONDITIONAL @@ -10511,7 +11163,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_P PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); - if (attr == NULL) goto pop_3_error; + if (attr == NULL) { + goto pop_3_error; + } attr_st = PyStackRef_FromPyObjectSteal(attr); stack_pointer[-3] = attr_st; stack_pointer += -2; @@ -10580,11 +11234,17 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL if (method_found) { self_or_null = self_st; // transfer ownership } else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(self_st); + stack_pointer = _PyFrame_GetStackPointer(frame); self_or_null = PyStackRef_NULL; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; @@ -10656,12 +11316,20 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS) PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(codeobj_st); - if (func_obj == NULL) goto pop_1_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (func_obj == NULL) { + goto error; + } _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); - stack_pointer[-1] = func; + stack_pointer[0] = func; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); pop_4_error: @@ -10704,7 +11372,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS) { PyStackRef_AsPyObjectSteal(value) ); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto pop_2_error; + if (err != 0) { + goto pop_2_error; + } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); } @@ -10756,7 +11426,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS) { attrs = PyStackRef_FromPyObjectSteal(attrs_o); } else { - if (_PyErr_Occurred(tstate)) goto pop_3_error; + if (_PyErr_Occurred(tstate)) { + goto pop_3_error; + } // Error! attrs = PyStackRef_None; // Failure! } @@ -10798,7 +11470,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS) { PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (values_or_none_o == NULL) goto error; + if (values_or_none_o == NULL) { + goto error; + } values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -11427,7 +12101,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS) { _Py_CODEUNIT *bytecode = _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) goto error; + if (bytecode == NULL) { + goto error; + } _PyFrame_SetStackPointer(frame, stack_pointer); ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11474,7 +12150,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS) { _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + if (err != 0) { + goto error; + } } } } @@ -11560,7 +12238,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAM _PyFrame_SetStackPointer(frame, stack_pointer); PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); stack_pointer = _PyFrame_GetStackPointer(frame); - if (gen == NULL) goto error; + if (gen == NULL) { + goto error; + } assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; @@ -11730,10 +12410,16 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS) { goto pop_1_error; } } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); retval = PyStackRef_FromPyObjectSteal(retval_o); } - stack_pointer[-1] = retval; + stack_pointer[0] = retval; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } DISPATCH(); pop_4_error: @@ -11854,18 +12540,24 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARA _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) goto error; + if (err < 0) { + goto error; + } if (ann_dict == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); ann_dict = PyDict_New(); stack_pointer = _PyFrame_GetStackPointer(frame); - if (ann_dict == NULL) goto error; + if (ann_dict == NULL) { + goto error; + } _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); Py_DECREF(ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; + if (err) { + goto error; + } } else { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -11906,7 +12598,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS) { PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); - if (err) goto pop_1_error; + if (err) { + goto pop_1_error; + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } @@ -11985,7 +12679,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS) { PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); - if (err < 0) goto pop_1_error; + if (err < 0) { + goto pop_1_error; + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } @@ -12047,7 +12743,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS) { stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); PyStackRef_CLOSE(owner); - if (err) goto pop_2_error; + if (err) { + goto pop_2_error; + } } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -12133,10 +12831,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_C _PyDictValues_AddToInsertionOrder(values, index); } UNLOCK_OBJECT(owner_o); - PyStackRef_CLOSE(owner); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); Py_XDECREF(old_value); stack_pointer = _PyFrame_GetStackPointer(frame); } @@ -12197,10 +12895,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS PyObject *old_value = *(PyObject **)addr; FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(owner_o); - PyStackRef_CLOSE(owner); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); Py_XDECREF(old_value); stack_pointer = _PyFrame_GetStackPointer(frame); } @@ -12310,10 +13008,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. STAT_INC(STORE_ATTR, hit); - PyStackRef_CLOSE(owner); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); Py_XDECREF(old_value); stack_pointer = _PyFrame_GetStackPointer(frame); } @@ -12480,7 +13178,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS) { int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); - if (err) goto pop_1_error; + if (err) { + goto pop_1_error; + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } @@ -12532,7 +13232,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS) { stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(v); - if (err) goto pop_1_error; + if (err) { + goto pop_1_error; + } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } @@ -12597,7 +13299,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS) { } PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); - if (err) goto pop_4_error; + if (err) { + goto pop_4_error; + } } stack_pointer += -4; assert(WITHIN_STACK_BOUNDS()); @@ -12660,7 +13364,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS) { PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); - if (err) goto pop_3_error; + if (err) { + goto pop_3_error; + } } stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); @@ -12710,10 +13416,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARA PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(value)); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(dict_st); - if (err) goto pop_3_error; stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } } DISPATCH(); pop_4_error: @@ -12787,10 +13497,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_ assert(old_value != NULL); UNLOCK_OBJECT(list); // unlock before decrefs! PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(list_st); Py_DECREF(old_value); stack_pointer = _PyFrame_GetStackPointer(frame); } @@ -12818,17 +13528,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SWAP); - _PyStackRef bottom_in; - _PyStackRef top_in; - _PyStackRef top_out; - _PyStackRef bottom_out; - top_in = stack_pointer[-1]; - bottom_in = stack_pointer[-2 - (oparg-2)]; - bottom_out = bottom_in; - top_out = top_in; + _PyStackRef *bottom; + _PyStackRef *top; + top = &stack_pointer[-1]; + bottom = &stack_pointer[-2 - (oparg-2)]; + _PyStackRef temp = bottom[0]; + bottom[0] = top[0]; + top[0] = temp; assert(oparg >= 2); - stack_pointer[-2 - (oparg-2)] = top_out; - stack_pointer[-1] = bottom_out; } DISPATCH(); pop_4_error: @@ -12883,7 +13590,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS) { int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (err < 0) goto pop_1_error; + if (err < 0) { + goto pop_1_error; + } res = err ? PyStackRef_True : PyStackRef_False; } stack_pointer[-1] = res; @@ -13186,7 +13895,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS) { PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (res_o == NULL) goto pop_1_error; + if (res_o == NULL) { + goto pop_1_error; + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; } @@ -13221,7 +13932,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS) PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); - if (res_o == NULL) goto pop_1_error; + if (res_o == NULL) { + goto pop_1_error; + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; } @@ -13290,7 +14003,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS) { int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); - if (res == 0) goto pop_1_error; + if (res == 0) { + goto pop_1_error; + } stack_pointer += (oparg & 0xFF) + (oparg >> 8); assert(WITHIN_STACK_BOUNDS()); } @@ -13350,7 +14065,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); - if (res == 0) goto pop_1_error; + if (res == 0) { + goto pop_1_error; + } } stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -13582,7 +14299,9 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARA PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) goto error; + if (res_o == NULL) { + goto error; + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; diff --git a/Python/generated_tail_call_labels.c.h b/Python/generated_tail_call_labels.c.h index 32f8ffc7c281c3..a8ba2901a9da9b 100644 --- a/Python/generated_tail_call_labels.c.h +++ b/Python/generated_tail_call_labels.c.h @@ -10,20 +10,20 @@ pop_4_error: { - STACK_SHRINK(1); - goto pop_3_error; + STACK_SHRINK(4); + goto error; } pop_3_error: { - STACK_SHRINK(1); - goto pop_2_error; + STACK_SHRINK(3); + goto error; } pop_2_error: { - STACK_SHRINK(1); - goto pop_1_error; + STACK_SHRINK(2); + goto error; } pop_1_error: From cf18981f3771dec28ab2bfba22aaffbc6b06130e Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 4 Feb 2025 00:11:46 +0800 Subject: [PATCH 099/110] Remove opcode --- Lib/test/test_generated_cases.py | 6 +- Python/ceval.c | 2 +- Python/ceval_macros.h | 11 +- Python/generated_tail_call_handlers.c.h | 897 +++++++++++++----- Python/generated_tail_call_labels.c.h | 4 +- Tools/cases_generator/tier1_generator.py | 6 +- .../tier1_tail_call_generator.py | 27 +- 7 files changed, 699 insertions(+), 254 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 307e087ad999a1..3ec9224a4690b2 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1886,6 +1886,8 @@ def test_basic(self): output = """ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_OP(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1927,6 +1929,8 @@ def test_label_transformed(self): Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_OP(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1944,7 +1948,7 @@ def test_label_transformed(self): if (x) { goto baz; } - return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); + return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0); } """ self.run_cases_test(input, output, output_labels) diff --git a/Python/ceval.c b/Python/ceval.c index 4801932603f0c2..6ff46c326f5b77 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -772,7 +772,7 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) static inline PyObject * _TAIL_CALL_entry(TAIL_CALL_PARAMS) { - opcode = next_instr->op.code; + int opcode = next_instr->op.code; oparg = next_instr->op.arg; return (INSTRUCTION_TABLE[opcode])(TAIL_CALL_ARGS); } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 8d4ad2dbe334cd..60c4b42f3e650d 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -70,8 +70,8 @@ #define INSTRUCTION_STATS(op) ((void)0) #endif -#define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int opcode, int oparg -#define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, opcode, oparg +#define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg +#define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, oparg #ifdef Py_TAIL_CALL_INTERP // Note: [[clang::musttail]] works for GCC 15, but not __attribute__((musttail)) at the moment. @@ -125,8 +125,15 @@ do { \ /* Do interpreter dispatch accounting for tracing and instrumentation */ +#ifdef Py_TAIL_CALL_INTERP +# define DEFINE_OPCODE_IF_NEEDED() int opcode; +#else +# define DEFINE_OPCODE_IF_NEEDED() +#endif + #define DISPATCH() \ { \ + DEFINE_OPCODE_IF_NEEDED(); \ assert(frame->stackpointer == NULL); \ NEXTOPARG(); \ PRE_DISPATCH_GOTO(); \ diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h index 78e5d20115acfb..bcee0e6b9ca525 100644 --- a/Python/generated_tail_call_handlers.c.h +++ b/Python/generated_tail_call_handlers.c.h @@ -169,6 +169,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP); @@ -239,6 +241,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 6; @@ -256,12 +260,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PA if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); } if (!PyFloat_CheckExact(right_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); } } /* Skip 5 cache entries */ @@ -306,6 +310,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 6; @@ -323,12 +329,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARA if (!PyLong_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); } if (!PyLong_CheckExact(right_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); } } /* Skip 5 cache entries */ @@ -372,6 +378,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 6; @@ -389,12 +397,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_ if (!PyUnicode_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); } if (!PyUnicode_CheckExact(right_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); } } /* Skip 5 cache entries */ @@ -438,6 +446,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 6; @@ -463,7 +473,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAM if (!res) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); } } /* Skip -4 cache entry */ @@ -507,6 +517,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 6; @@ -523,12 +535,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TA if (!PyUnicode_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); } if (!PyUnicode_CheckExact(right_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); } } /* Skip 5 cache entries */ @@ -549,7 +561,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TA if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. @@ -603,6 +615,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 6; @@ -620,12 +634,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CA if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); } if (!PyFloat_CheckExact(right_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); } } /* Skip 5 cache entries */ @@ -670,6 +684,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 6; @@ -687,12 +703,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL if (!PyLong_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); } if (!PyLong_CheckExact(right_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); } } /* Skip 5 cache entries */ @@ -736,6 +752,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 6; @@ -753,12 +771,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CA if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); } if (!PyFloat_CheckExact(right_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); } } /* Skip 5 cache entries */ @@ -803,6 +821,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 6; @@ -820,12 +840,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL if (!PyLong_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); } if (!PyLong_CheckExact(right_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); } } /* Skip 5 cache entries */ @@ -869,6 +889,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BINARY_SLICE); @@ -943,6 +965,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR); @@ -1010,6 +1034,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -1026,7 +1052,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PAR if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o; @@ -1070,6 +1096,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PAR Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -1085,7 +1113,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } } // _BINARY_SUBSCR_CHECK_FUNC @@ -1095,28 +1123,28 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_ if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); if (getitem_o == NULL) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } assert(PyFunction_Check(getitem_o)); uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); @@ -1168,6 +1196,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -1184,18 +1214,18 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL if (!PyLong_CheckExact(sub)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } if (!PyList_CheckExact(list)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } // Deopt unless 0 <= sub < PyList_Size(list) if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; #ifdef Py_GIL_DISABLED @@ -1205,14 +1235,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL if (res_o == NULL) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(BINARY_SUBSCR, hit); #else if (index >= PyList_GET_SIZE(list)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyList_GET_ITEM(list, index); @@ -1251,6 +1281,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -1267,30 +1299,30 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_ if (!PyLong_CheckExact(sub)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } if (!PyUnicode_CheckExact(str)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (PyUnicode_GET_LENGTH(str) <= index) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; @@ -1326,6 +1358,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -1342,24 +1376,24 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CAL if (!PyLong_CheckExact(sub)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } if (!PyTuple_CheckExact(tuple)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } // Deopt unless 0 <= sub < PyTuple_Size(list) if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (index >= PyTuple_GET_SIZE(tuple)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyTuple_GET_ITEM(tuple, index); @@ -1397,6 +1431,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CAL Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_LIST); @@ -1433,6 +1469,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_MAP); @@ -1489,6 +1527,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SET); @@ -1551,6 +1591,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SLICE); @@ -1595,6 +1637,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_STRING); @@ -1646,6 +1690,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_TUPLE); @@ -1682,6 +1728,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CACHE); @@ -1709,6 +1757,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL); @@ -1891,6 +1941,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -1909,7 +1961,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_C if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } } // _CHECK_AND_ALLOCATE_OBJECT @@ -1924,18 +1976,18 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_C if (!PyStackRef_IsNull(null[0])) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } if (!PyType_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } PyTypeObject *tp = (PyTypeObject *)callable_o; if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } assert(tp->tp_new == PyBaseObject_Type.tp_new); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); @@ -1946,7 +1998,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_C if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2030,6 +2082,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_C Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -2046,7 +2100,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS @@ -2056,12 +2110,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI if (!PyStackRef_IsNull(null[0])) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS @@ -2085,13 +2139,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } } // _CHECK_FUNCTION_EXACT_ARGS @@ -2103,7 +2157,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } } // _CHECK_STACK_SPACE @@ -2114,12 +2168,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } } // _INIT_CALL_PY_EXACT_ARGS @@ -2182,6 +2236,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAI Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -2198,7 +2254,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_C if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } } // _CHECK_METHOD_VERSION @@ -2210,23 +2266,23 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_C if (Py_TYPE(callable_o) != &PyMethod_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } PyObject *func = ((PyMethodObject *)callable_o)->im_func; if (!PyFunction_Check(func)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } if (((PyFunctionObject *)func)->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } if (!PyStackRef_IsNull(null[0])) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } } // _EXPAND_METHOD @@ -2317,6 +2373,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_C Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -2337,7 +2395,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR if (!PyType_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } PyTypeObject *tp = (PyTypeObject *)callable_o; int total_args = oparg; @@ -2349,7 +2407,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR if (tp->tp_vectorcall == NULL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -2422,6 +2480,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -2449,12 +2509,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARA if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); @@ -2533,6 +2593,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -2560,12 +2622,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS( if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(CALL, hit); /* res = func(self, arguments, nargs, kwnames) */ @@ -2645,6 +2707,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS( Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -2671,23 +2735,23 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS) if (total_args != 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } // CPython promises to check all non-vectorcall function calls. if (tstate->c_recursion_remaining <= 0) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); @@ -2754,6 +2818,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; @@ -2957,6 +3023,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_1); @@ -2995,6 +3063,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_2); @@ -3040,6 +3110,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -3065,13 +3137,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS if (total_args != 2) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } PyInterpreterState *interp = tstate->interp; if (callable_o != interp->callable_cache.isinstance) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(CALL, hit); _PyStackRef cls_stackref = arguments[1]; @@ -3114,6 +3186,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW); @@ -3295,6 +3369,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -3312,7 +3388,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_P if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, oparg); } } // _CHECK_METHOD_VERSION_KW @@ -3324,23 +3400,23 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_P if (Py_TYPE(callable_o) != &PyMethod_Type) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, oparg); } PyObject *func = ((PyMethodObject *)callable_o)->im_func; if (!PyFunction_Check(func)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, oparg); } if (((PyFunctionObject *)func)->func_version != func_version) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, oparg); } if (!PyStackRef_IsNull(null[0])) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, oparg); } } // _EXPAND_METHOD_KW @@ -3441,6 +3517,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_P Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -3461,12 +3539,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) if (PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, oparg); } if (Py_TYPE(callable_o) == &PyMethod_Type) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, oparg); } } // _CALL_KW_NON_PY @@ -3567,6 +3645,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -3583,7 +3663,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS) { if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, oparg); } } // _CHECK_FUNCTION_VERSION_KW @@ -3594,13 +3674,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS) { if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, oparg); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, oparg); } } // _PY_FRAME_KW @@ -3687,6 +3767,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -3711,13 +3793,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS) { if (total_args != 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } PyInterpreterState *interp = tstate->interp; if (callable_o != interp->callable_cache.len) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(CALL, hit); _PyStackRef arg_stackref = args[0]; @@ -3767,6 +3849,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -3787,18 +3871,18 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAM if (callable_o != interp->callable_cache.list_append) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } assert(self_o != NULL); if (!PyList_Check(self_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } if (!LOCK_OBJECT(self_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); @@ -3844,6 +3928,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -3872,19 +3958,19 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(CALL, hit); int nargs = total_args - 1; @@ -3961,6 +4047,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -3988,20 +4076,20 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } PyTypeObject *d_type = method->d_common.d_type; PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); if (!Py_IS_TYPE(self, d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(CALL, hit); int nargs = total_args - 1; @@ -4078,6 +4166,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -4104,13 +4194,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TA if (total_args != 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } PyMethodDef *meth = method->d_method; _PyStackRef self_stackref = args[0]; @@ -4118,18 +4208,18 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TA if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } if (meth->ml_flags != METH_NOARGS) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } // CPython promises to check all non-vectorcall function calls. if (tstate->c_recursion_remaining <= 0) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; @@ -4195,6 +4285,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -4222,24 +4314,24 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CA if (total_args != 2) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } // CPython promises to check all non-vectorcall function calls. if (tstate->c_recursion_remaining <= 0) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } _PyStackRef arg_stackref = arguments[1]; _PyStackRef self_stackref = arguments[0]; @@ -4247,7 +4339,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CA method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; @@ -4314,6 +4406,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -4333,12 +4427,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PA if (PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } if (Py_TYPE(callable_o) == &PyMethod_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } } // _CALL_NON_PY_GENERAL @@ -4430,6 +4524,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -4445,7 +4541,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PAR if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } } // _CHECK_FUNCTION_VERSION @@ -4456,13 +4552,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PAR if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } } // _CHECK_FUNCTION_EXACT_ARGS @@ -4475,7 +4571,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PAR if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } } // _CHECK_STACK_SPACE @@ -4486,12 +4582,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PAR if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } } // _INIT_CALL_PY_EXACT_ARGS @@ -4554,6 +4650,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PAR Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -4569,7 +4667,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } } // _CHECK_FUNCTION_VERSION @@ -4580,13 +4678,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } } // _PY_FRAME_GENERAL @@ -4663,6 +4761,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -4685,12 +4785,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS) { if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } if (callable_o != (PyObject *)&PyUnicode_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4749,6 +4849,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -4771,12 +4873,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS) { if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } if (callable_o != (PyObject *)&PyTuple_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4835,6 +4937,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -4855,12 +4959,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS) { if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } if (callable_o != (PyObject *)&PyType_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); @@ -4892,6 +4996,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EG_MATCH); @@ -4961,6 +5067,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EXC_MATCH); @@ -5007,6 +5115,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; @@ -5067,6 +5177,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP); @@ -5150,6 +5262,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -5167,12 +5281,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAM if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, oparg); } if (!PyFloat_CheckExact(right_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, oparg); } } /* Skip 1 cache entry */ @@ -5215,6 +5329,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -5232,12 +5348,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS) if (!PyLong_CheckExact(left_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, oparg); } if (!PyLong_CheckExact(right_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, oparg); } } /* Skip 1 cache entry */ @@ -5248,12 +5364,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS) if (!_PyLong_IsCompact((PyLongObject *)left_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, oparg); } if (!_PyLong_IsCompact((PyLongObject *)right_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && @@ -5292,6 +5408,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -5309,12 +5427,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS) if (!PyUnicode_CheckExact(left_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, oparg); } if (!PyUnicode_CheckExact(right_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, oparg); } } /* Skip 1 cache entry */ @@ -5358,6 +5476,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP); @@ -5424,6 +5544,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -5440,7 +5562,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAM if (!PyDict_CheckExact(right_o)) { UPDATE_MISS_STATS(CONTAINS_OP); assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); - Py_MUSTTAIL return _TAIL_CALL_CONTAINS_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CONTAINS_OP(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5477,6 +5599,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -5493,7 +5617,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS if (!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o))) { UPDATE_MISS_STATS(CONTAINS_OP); assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); - Py_MUSTTAIL return _TAIL_CALL_CONTAINS_OP(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_CONTAINS_OP(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! @@ -5531,6 +5655,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CONVERT_VALUE); @@ -5577,6 +5703,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY); @@ -5610,6 +5738,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY_FREE_VARS); @@ -5646,6 +5776,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_ATTR); @@ -5683,6 +5815,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_DEREF); @@ -5721,6 +5855,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_FAST); @@ -5757,6 +5893,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_GLOBAL); @@ -5797,6 +5935,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_NAME); @@ -5844,6 +5984,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_SUBSCR); @@ -5885,6 +6027,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_MERGE); @@ -5932,6 +6076,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_UPDATE); @@ -5983,6 +6129,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; @@ -6032,6 +6180,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); next_instr += 1; INSTRUCTION_STATS(END_FOR); _PyStackRef value; @@ -6068,6 +6218,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_SEND); @@ -6104,6 +6256,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; @@ -6156,6 +6310,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXIT_INIT_CHECK); @@ -6194,6 +6350,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXTENDED_ARG); @@ -6224,6 +6382,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_SIMPLE); @@ -6277,6 +6437,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_WITH_SPEC); @@ -6319,6 +6481,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER); @@ -6399,6 +6563,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -6413,7 +6579,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS) { if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, oparg); } } // _FOR_ITER_GEN_FRAME @@ -6423,12 +6589,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS) { if (Py_TYPE(gen) != &PyGen_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, oparg); } if (gen->gi_frame_state >= FRAME_EXECUTING) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(FOR_ITER, hit); gen_frame = &gen->gi_iframe; @@ -6478,6 +6644,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -6492,7 +6660,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS) if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, oparg); } } // _ITER_JUMP_LIST @@ -6552,6 +6720,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -6567,7 +6737,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS) if (Py_TYPE(r) != &PyRangeIter_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, oparg); } } // _ITER_JUMP_RANGE @@ -6620,6 +6790,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -6634,7 +6806,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS) if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, oparg); } } // _ITER_JUMP_TUPLE @@ -6691,6 +6863,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AITER); @@ -6758,6 +6932,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ANEXT); @@ -6796,6 +6972,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AWAITABLE); @@ -6833,6 +7011,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ITER); @@ -6871,6 +7051,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_LEN); @@ -6914,6 +7096,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_YIELD_FROM_ITER); @@ -6975,6 +7159,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_FROM); @@ -7014,6 +7200,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_NAME); @@ -7059,6 +7247,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -7251,6 +7441,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; @@ -7454,6 +7646,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -7643,6 +7837,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_P Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; next_instr += 1; @@ -7686,6 +7882,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_P Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; @@ -7733,6 +7931,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -7788,6 +7988,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; @@ -7828,6 +8030,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -7872,6 +8076,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; @@ -7899,6 +8105,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_C Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -7956,6 +8164,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -8063,6 +8273,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAI Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -8092,6 +8304,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -8127,6 +8341,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -8161,6 +8377,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(T Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -8199,6 +8417,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -8235,6 +8455,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NON Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -8269,6 +8491,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; @@ -8365,6 +8589,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; @@ -8428,6 +8654,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_C Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; @@ -8513,6 +8741,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INTERPRETER_EXIT); @@ -8550,6 +8780,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IS_OP); @@ -8587,6 +8819,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD); @@ -8650,6 +8884,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -8736,6 +8972,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); @@ -8768,6 +9006,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); @@ -8818,6 +9058,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_P Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_FORWARD); @@ -8844,6 +9086,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_APPEND); @@ -8880,6 +9124,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_EXTEND); @@ -8935,6 +9181,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR); @@ -9033,6 +9281,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 10; @@ -9050,13 +9300,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS if (!PyType_Check(owner_o)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } /* Skip 2 cache entries */ @@ -9098,6 +9348,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 10; @@ -9115,13 +9367,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_C if (!PyType_Check(owner_o)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } // _GUARD_TYPE_VERSION @@ -9132,7 +9384,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_C if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } // _LOAD_ATTR_CLASS @@ -9173,6 +9425,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_C Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 10; @@ -9189,14 +9443,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDE if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; @@ -9204,14 +9458,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDE if (f->func_version != func_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 2); if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); @@ -9244,6 +9498,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDE Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 10; @@ -9262,7 +9518,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } // _CHECK_MANAGED_OBJECT_HAS_VALUES @@ -9273,7 +9529,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } // _LOAD_ATTR_INSTANCE_VALUE @@ -9285,14 +9541,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } #ifdef Py_GIL_DISABLED if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } #else @@ -9334,6 +9590,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 10; @@ -9352,7 +9610,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } // _CHECK_ATTR_METHOD_LAZY_DICT @@ -9364,7 +9622,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_ if (dict != NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } /* Skip 1 cache entry */ @@ -9404,6 +9662,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 10; @@ -9422,7 +9682,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CA if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } /* Skip 2 cache entries */ @@ -9463,6 +9723,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 10; @@ -9481,7 +9743,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAI if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT @@ -9492,7 +9754,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAI if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } // _GUARD_KEYS_VERSION @@ -9504,7 +9766,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAI if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } // _LOAD_ATTR_METHOD_WITH_VALUES @@ -9544,6 +9806,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAI Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 10; @@ -9562,7 +9826,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); @@ -9570,7 +9834,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } mod_keys = keys; } @@ -9585,7 +9849,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); @@ -9593,7 +9857,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } #else @@ -9636,6 +9900,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 10; @@ -9653,7 +9919,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT( if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } /* Skip 2 cache entries */ @@ -9690,6 +9956,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT( Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 10; @@ -9707,7 +9975,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VAL if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT @@ -9718,7 +9986,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VAL if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } // _GUARD_KEYS_VERSION @@ -9730,7 +9998,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VAL if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES @@ -9765,6 +10033,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VAL Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 10; @@ -9778,7 +10048,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } // _GUARD_TYPE_VERSION @@ -9790,7 +10060,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } /* Skip 2 cache entries */ @@ -9804,22 +10074,22 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } if (code->co_kwonlyargcount) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } if (code->co_argcount != 1) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(LOAD_ATTR, hit); new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); @@ -9873,6 +10143,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PAR Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 10; @@ -9891,7 +10163,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } // _LOAD_ATTR_SLOT @@ -9903,14 +10175,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); if (!increfed) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } #else attr = PyStackRef_FromPyObjectNew(attr_o); @@ -9949,6 +10221,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 10; @@ -9968,7 +10242,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } // _CHECK_ATTR_WITH_HINT @@ -9979,7 +10253,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA if (dict_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } assert(PyDict_CheckExact((PyObject *)dict_o)); dict = dict_o; @@ -9992,7 +10266,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } if (hint >= (size_t)dict->ma_keys->dk_nentries) { @@ -10000,7 +10274,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); @@ -10009,7 +10283,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; @@ -10018,7 +10292,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } attr_o = ep->me_value; @@ -10027,7 +10301,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } STAT_INC(LOAD_ATTR, hit); @@ -10066,6 +10340,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_BUILD_CLASS); @@ -10110,6 +10386,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); @@ -10150,6 +10428,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_P Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); @@ -10201,6 +10481,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST_IMMORTAL); @@ -10234,6 +10516,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_PA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_MORTAL(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST_MORTAL); @@ -10266,6 +10550,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_MORTAL(TAIL_CALL_PARA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_DEREF); @@ -10304,6 +10590,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST); @@ -10335,6 +10623,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); @@ -10367,6 +10657,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_CHECK); @@ -10407,6 +10699,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); @@ -10442,6 +10736,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); @@ -10501,6 +10797,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CAL Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); @@ -10595,6 +10893,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_C Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL); @@ -10663,6 +10963,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 5; @@ -10679,13 +10981,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, oparg); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, oparg); } assert(DK_IS_UNICODE(keys)); } @@ -10696,13 +10998,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, oparg); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, oparg); } builtins_keys = keys; assert(DK_IS_UNICODE(builtins_keys)); @@ -10715,14 +11017,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA if (res_o == NULL) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, oparg); } #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); if (!increfed) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, oparg); } #else Py_INCREF(res_o); @@ -10760,6 +11062,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 5; @@ -10776,13 +11080,13 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PAR if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, oparg); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, oparg); } globals_keys = keys; assert(DK_IS_UNICODE(globals_keys)); @@ -10796,14 +11100,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PAR if (res_o == NULL) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, oparg); } #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); if (!increfed) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, oparg); } #else Py_INCREF(res_o); @@ -10841,6 +11145,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PAR Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_LOCALS); @@ -10879,6 +11185,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_NAME); @@ -10916,6 +11224,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SMALL_INT); @@ -10948,6 +11258,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SPECIAL); @@ -11003,6 +11315,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR); @@ -11128,6 +11442,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -11148,12 +11464,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_P if (global_super != (PyObject *)&PySuper_Type) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_SUPER_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_SUPER_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } if (!PyType_Check(class)) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_SUPER_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_SUPER_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); @@ -11192,6 +11508,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_P Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -11213,12 +11531,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL if (global_super != (PyObject *)&PySuper_Type) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_SUPER_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_SUPER_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } if (!PyType_Check(class)) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_SUPER_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_LOAD_SUPER_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); @@ -11272,6 +11590,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_CELL); @@ -11305,6 +11625,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_FUNCTION); @@ -11352,6 +11674,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAP_ADD); @@ -11399,6 +11723,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_CLASS); @@ -11457,6 +11783,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_KEYS); @@ -11499,6 +11827,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_MAPPING); @@ -11532,6 +11862,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_SEQUENCE); @@ -11565,6 +11897,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOP); @@ -11590,6 +11924,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOT_TAKEN); @@ -11615,6 +11951,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_EXCEPT); @@ -11650,6 +11988,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_ITER); @@ -11680,6 +12020,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -11715,6 +12057,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -11766,6 +12110,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -11817,6 +12163,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_P Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -11852,6 +12200,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_TOP); @@ -11882,6 +12232,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_EXC_INFO); @@ -11925,6 +12277,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_NULL); @@ -11955,6 +12309,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; @@ -11998,6 +12354,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; @@ -12059,6 +12417,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESERVED); @@ -12086,6 +12446,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME); @@ -12178,6 +12540,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; @@ -12187,7 +12551,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS) { if (_Py_emscripten_signal_clock == 0) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); - Py_MUSTTAIL return _TAIL_CALL_RESUME(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_RESUME(frame, stack_pointer, tstate, this_instr, oparg); } _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif @@ -12197,14 +12561,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS) { if (eval_breaker != version) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); - Py_MUSTTAIL return _TAIL_CALL_RESUME(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_RESUME(frame, stack_pointer, tstate, this_instr, oparg); } #ifdef Py_GIL_DISABLED if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); - Py_MUSTTAIL return _TAIL_CALL_RESUME(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_RESUME(frame, stack_pointer, tstate, this_instr, oparg); } #endif } @@ -12229,6 +12593,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); @@ -12282,6 +12648,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_VALUE); @@ -12328,6 +12696,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND); @@ -12442,6 +12812,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -12457,7 +12829,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS) { if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); - Py_MUSTTAIL return _TAIL_CALL_SEND(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_SEND(frame, stack_pointer, tstate, this_instr, oparg); } } // _SEND_GEN_FRAME @@ -12468,12 +12840,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS) { if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); - Py_MUSTTAIL return _TAIL_CALL_SEND(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_SEND(frame, stack_pointer, tstate, this_instr, oparg); } if (gen->gi_frame_state >= FRAME_EXECUTING) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); - Py_MUSTTAIL return _TAIL_CALL_SEND(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_SEND(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(SEND, hit); gen_frame = &gen->gi_iframe; @@ -12525,6 +12897,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SETUP_ANNOTATIONS); @@ -12586,6 +12960,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_ADD); @@ -12625,6 +13001,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); @@ -12667,6 +13045,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_UPDATE); @@ -12706,6 +13086,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR); @@ -12771,6 +13153,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 5; @@ -12788,7 +13172,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_C if (!LOCK_OBJECT(owner_o)) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } PyTypeObject *tp = Py_TYPE(owner_o); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { @@ -12796,7 +13180,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_C if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } } @@ -12811,7 +13195,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_C if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } } @@ -12860,6 +13244,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_C Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 5; @@ -12877,7 +13263,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } // _STORE_ATTR_SLOT @@ -12888,7 +13274,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS if (!LOCK_OBJECT(owner_o)) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); @@ -12924,6 +13310,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 5; @@ -12941,7 +13329,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } // _STORE_ATTR_WITH_HINT @@ -12954,12 +13342,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P if (dict == NULL) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } if (!LOCK_OBJECT(dict)) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } #ifdef Py_GIL_DISABLED if (dict != _PyObject_GetManagedDict(owner_o)) { @@ -12967,7 +13355,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } #endif @@ -12979,7 +13367,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; @@ -12988,7 +13376,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } PyObject *old_value = ep->me_value; @@ -12997,7 +13385,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); } } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -13037,6 +13425,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_P Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_DEREF); @@ -13070,6 +13460,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST); @@ -13100,6 +13492,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); @@ -13133,6 +13527,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_P Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_STORE_FAST); @@ -13168,6 +13564,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_GLOBAL); @@ -13205,6 +13603,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_NAME); @@ -13259,6 +13659,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_SLICE); @@ -13327,6 +13729,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR); @@ -13392,6 +13796,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -13408,7 +13814,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARA if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(STORE_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -13446,6 +13852,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -13463,24 +13871,24 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_ if (!PyLong_CheckExact(sub)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } if (!PyList_CheckExact(list)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } // Ensure nonnegative, zero-or-one-digit ints. if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (!LOCK_OBJECT(list)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } // Ensure index < len(list) if (index >= PyList_GET_SIZE(list)) { @@ -13488,7 +13896,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_ if (true) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); } } STAT_INC(STORE_SUBSCR, hit); @@ -13525,6 +13933,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SWAP); @@ -13558,6 +13968,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL); @@ -13618,6 +14030,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -13636,7 +14050,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PA if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, oparg); } } // _REPLACE_WITH_TRUE @@ -13668,6 +14082,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -13680,7 +14096,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS) { if (!PyStackRef_BoolCheck(value)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(TO_BOOL, hit); } @@ -13705,6 +14121,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -13719,7 +14137,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS) { if (!PyLong_CheckExact(value_o)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value_o)) { @@ -13753,6 +14171,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -13767,7 +14187,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS) { if (!PyList_CheckExact(value_o)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(TO_BOOL, hit); res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; @@ -13795,6 +14215,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -13809,7 +14231,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS) { if (!PyStackRef_IsNone(value)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(TO_BOOL, hit); res = PyStackRef_False; @@ -13836,6 +14258,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; @@ -13850,7 +14274,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS) { if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(TO_BOOL, hit); if (value_o == &_Py_STR(empty)) { @@ -13885,6 +14309,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_INVERT); @@ -13922,6 +14348,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NEGATIVE); @@ -13959,6 +14387,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS) Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NOT); @@ -13991,6 +14421,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNPACK_EX); @@ -14030,6 +14462,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS) { Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE); @@ -14093,6 +14527,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -14107,19 +14543,19 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_P if (!PyList_CheckExact(seq_o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, oparg); } if (!LOCK_OBJECT(seq_o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, oparg); } if (PyList_GET_SIZE(seq_o) != oparg) { UNLOCK_OBJECT(seq_o); if (true) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, oparg); } } STAT_INC(UNPACK_SEQUENCE, hit); @@ -14153,6 +14589,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_P Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -14167,12 +14605,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_ if (!PyTuple_CheckExact(seq_o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, oparg); } if (PyTuple_GET_SIZE(seq_o) != oparg) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq_o); @@ -14204,6 +14642,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; @@ -14219,12 +14659,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_C if (!PyTuple_CheckExact(seq_o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, oparg); } if (PyTuple_GET_SIZE(seq_o) != 2) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, opcode, oparg); + Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, oparg); } STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); @@ -14256,6 +14696,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_C Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(WITH_EXCEPT_START); @@ -14328,6 +14770,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(YIELD_VALUE); @@ -14392,6 +14836,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS) { /* END INSTRUCTIONS */ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { { + int opcode = next_instr->op.code; _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", diff --git a/Python/generated_tail_call_labels.c.h b/Python/generated_tail_call_labels.c.h index a8ba2901a9da9b..4671013ec0f810 100644 --- a/Python/generated_tail_call_labels.c.h +++ b/Python/generated_tail_call_labels.c.h @@ -103,7 +103,7 @@ lltrace_resume_frame(frame); } #endif - return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); + return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0); } exit_unwind: @@ -150,7 +150,7 @@ caller loses its exception */ assert(!_PyErr_Occurred(tstate)); #endif - return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0); + return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0); } #undef TIER_ONE diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 8a854e49eeb3e9..ed57d65eca956c 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -137,8 +137,12 @@ def write_single_inst( emitter: Emitter, name: str, inst: Instruction, - uses_this: Callable[[Instruction], bool] + uses_this: Callable[[Instruction], bool], + is_in_tail_call: bool = False ) -> None: + if is_in_tail_call: + out.emit("int opcode = next_instr->op.code;\n") + out.emit("(void)(opcode);\n") needs_this = uses_this(inst) unused_guard = "(void)this_instr;\n" if inst.properties.needs_prev: diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py index 8bd66e76edadb5..1e1993d17ff62d 100644 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ b/Tools/cases_generator/tier1_tail_call_generator.py @@ -54,23 +54,6 @@ def __init__(self, out: CWriter, analysis: Analysis): super().__init__(out) self.analysis = analysis - def go_to_instruction( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: Uop | Label, - storage: Storage, - inst: Instruction | None, - ) -> bool: - next(tkn_iter) - name = next(tkn_iter) - next(tkn_iter) - next(tkn_iter) - assert name.kind == "IDENTIFIER" - self.out.start_line() - self.emit(f"Py_MUSTTAIL return (INSTRUCTION_TABLE[{name.text}])(frame, stack_pointer, tstate, this_instr, opcode, oparg);\n") - return True - def deopt_if( self, tkn: Token, @@ -92,7 +75,8 @@ def deopt_if( family_name = inst.family.name self.emit(f"UPDATE_MISS_STATS({family_name});\n") self.emit(f"assert(_PyOpcode_Deopt[opcode] == ({family_name}));\n") - self.emit(f"Py_MUSTTAIL return _TAIL_CALL_{family_name}(frame, stack_pointer, tstate, this_instr, opcode, oparg);\n") + # Note it's not TAIL_CALL_ARGS, it passes this_instr instead of next_instr. + self.emit(f"Py_MUSTTAIL return _TAIL_CALL_{family_name}(frame, stack_pointer, tstate, this_instr, oparg);\n") self.emit("}\n") return not always_true(first_tkn) @@ -145,7 +129,7 @@ def dispatch( next(tkn_iter) next(tkn_iter) self.out.start_line() - self.emit("return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0, 0);\n") + self.emit("return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0);\n") return True @@ -165,7 +149,6 @@ def generate_label_handlers( for name, label in analysis.labels.items(): emitter.emit(f"{function_proto(name)}\n") emitter.emit_label(label) - emitter.emit("\n") emitter.emit("\n") @@ -195,6 +178,7 @@ def generate_tier1( emitter = TailCallCevalLabelsEmitter(out) generate_tier1_labels(analysis, emitter) labels_outfile.write(FOOTER) + write_header(__file__, filenames, outfile) outfile.write(PRELUDE) out = CWriter(outfile, 0, lines) @@ -214,7 +198,7 @@ def generate_tier1( # escaping locals. # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=118430#c1 out.emit("{\n") - write_single_inst(out, emitter, name, inst, uses_this) + write_single_inst(out, emitter, name, inst, uses_this, is_in_tail_call=True) out.emit("}\n") if not inst.parts[-1].properties.always_exits: out.emit("DISPATCH();\n") @@ -234,6 +218,7 @@ def generate_tier1( out.emit(function_proto("UNKNOWN_OPCODE")) out.emit(" {\n") out.emit("{\n") + out.emit("int opcode = next_instr->op.code;\n") out.emit(""" _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", From 1be3e2ab18dc89854a3cc0d51d461774d6e58f51 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 4 Feb 2025 04:36:13 +0800 Subject: [PATCH 100/110] Merge the two interpreters into one --- .gitattributes | 2 - .github/workflows/tail-call.yml | 4 - Lib/test/test_generated_cases.py | 160 +- Makefile.pre.in | 9 +- Python/bytecodes.c | 40 +- Python/ceval.c | 24 +- Python/ceval_macros.h | 48 +- Python/generated_cases.c.h | 2690 ++- Python/generated_tail_call_handlers.c.h | 15107 ---------------- Python/generated_tail_call_labels.c.h | 156 - Tools/c-analyzer/cpython/_parser.py | 2 - Tools/cases_generator/analyzer.py | 1 + Tools/cases_generator/generators_common.py | 8 +- Tools/cases_generator/tier1_generator.py | 199 +- .../tier1_tail_call_generator.py | 279 - 15 files changed, 2397 insertions(+), 16332 deletions(-) delete mode 100644 Python/generated_tail_call_handlers.c.h delete mode 100644 Python/generated_tail_call_labels.c.h delete mode 100644 Tools/cases_generator/tier1_tail_call_generator.py diff --git a/.gitattributes b/.gitattributes index 74d7c7f0bb97f9..2f5a030981fb94 100644 --- a/.gitattributes +++ b/.gitattributes @@ -95,8 +95,6 @@ Programs/test_frozenmain.h generated Python/Python-ast.c generated Python/executor_cases.c.h generated Python/generated_cases.c.h generated -Python/generated_tail_call_handlers.c.h generated -Python/generated_tail_call_labels.c.h generated Python/optimizer_cases.c.h generated Python/opcode_targets.h generated Python/stdlib_module_names.h generated diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index ed4fdea6ca05f4..ad5d0aa55173c4 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -5,15 +5,11 @@ on: - 'Python/bytecodes.c' - 'Python/ceval.c' - 'Python/ceval_macros.h' - - 'Python/generated_tail_call_handlers.c.h' - - 'Python/generated_tail_call_labels.c.h' push: paths: - 'Python/bytecodes.c' - 'Python/ceval.c' - 'Python/ceval_macros.h' - - 'Python/generated_tail_call_handlers.c.h' - - 'Python/generated_tail_call_labels.c.h' workflow_dispatch: permissions: diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 3ec9224a4690b2..b86f0afc8fc328 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -36,7 +36,6 @@ def skip_if_different_mount_drives(): import parser from stack import Local, Stack import tier1_generator - import tier1_tail_call_generator import opcode_metadata_generator import optimizer_generator @@ -462,7 +461,7 @@ def test_predictions(self): if (xxx) { UPDATE_MISS_STATS(OP1); assert(_PyOpcode_Deopt[opcode] == (OP1)); - goto PREDICTED_OP1; + JUMP_TO_PREDICTED(OP1); } res = Py_None; stack_pointer[-1] = res; @@ -544,7 +543,7 @@ def test_error_if_plain(self): next_instr += 1; INSTRUCTION_STATS(OP); if (cond) { - goto label; + JUMP_TO_LABEL(label); } DISPATCH(); } @@ -563,7 +562,7 @@ def test_error_if_plain_with_comment(self): next_instr += 1; INSTRUCTION_STATS(OP); if (cond) { - goto label; + JUMP_TO_LABEL(label); } // Comment is ok DISPATCH(); @@ -592,7 +591,7 @@ def test_error_if_pop(self): left = stack_pointer[-2]; SPAM(left, right); if (cond) { - goto pop_2_label; + JUMP_TO_LABEL(pop_2_label); } res = 0; stack_pointer[-2] = res; @@ -623,7 +622,7 @@ def test_error_if_pop_with_result(self): left = stack_pointer[-2]; res = SPAM(left, right); if (cond) { - goto pop_2_label; + JUMP_TO_LABEL(pop_2_label); } stack_pointer[-2] = res; stack_pointer += -1; @@ -640,8 +639,9 @@ def test_cache_effect(self): """ output = """ TARGET(OP) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(OP); uint16_t counter = read_u16(&this_instr[1].cache); @@ -726,8 +726,9 @@ def test_macro_instruction(self): } TARGET(OP1) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(OP1); _PyStackRef left; @@ -942,7 +943,7 @@ def test_array_error_if(self): if (oparg == 0) { stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto somewhere; + JUMP_TO_LABEL(somewhere); } stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1406,7 +1407,7 @@ def test_pop_on_error_peeks(self): { // Mark j and k as used if (cond) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } } stack_pointer += -2; @@ -1450,7 +1451,7 @@ def test_push_then_error(self): stack_pointer[1] = b; stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } } stack_pointer[0] = a; @@ -1477,14 +1478,14 @@ def test_error_if_true(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP1); - goto here; + JUMP_TO_LABEL(here); } TARGET(OP2) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP2); - goto there; + JUMP_TO_LABEL(there); } """ self.run_cases_test(input, output) @@ -1784,7 +1785,7 @@ def test_complex_label(self): """ output = """ - my_label: + LABEL(my_label) { // Comment do_thing() @@ -1812,146 +1813,21 @@ def test_multiple_labels(self): """ output = """ - my_label_1: + LABEL(my_label_1) { // Comment do_thing1(); goto my_label_2; } - my_label_2: + LABEL(my_label_2) { // Comment do_thing2(); goto my_label_3; } """ - -class TestGeneratedTailCallHandlers(unittest.TestCase): - def setUp(self) -> None: - super().setUp() - self.maxDiff = None - - self.temp_dir = tempfile.gettempdir() - self.temp_input_filename = os.path.join(self.temp_dir, "input.txt") - self.temp_output_filename = os.path.join(self.temp_dir, "output.txt") - self.temp_labels_output_filename = os.path.join(self.temp_dir, "labels_output.txt") - - def tearDown(self) -> None: - for filename in [ - self.temp_input_filename, - self.temp_output_filename, - self.temp_labels_output_filename, - ]: - try: - os.remove(filename) - except Exception: - pass - super().tearDown() - - def run_cases_test(self, input: str, expected: str, expected_labels: str): - with open(self.temp_input_filename, "w+") as temp_input: - temp_input.write(parser.BEGIN_MARKER) - temp_input.write(input) - temp_input.write(parser.END_MARKER) - temp_input.flush() - - with handle_stderr(): - tier1_tail_call_generator.generate_tier1_from_files( - [self.temp_input_filename], self.temp_output_filename, self.temp_labels_output_filename, False - ) - - with open(self.temp_output_filename) as temp_output: - lines = temp_output.read() - _, rest = lines.split(tier1_generator.INSTRUCTION_START_MARKER) - instructions, _ = rest.split(tier1_generator.INSTRUCTION_END_MARKER) - - with open(self.temp_labels_output_filename) as temp_output: - lines = temp_output.readlines() - while lines and lines[0].startswith(("// ", "#", " #", "\n")): - lines.pop(0) - while lines and lines[-1].startswith(("#", "\n")): - lines.pop(-1) - actual_labels = "".join(lines) - - self.assertEqual(instructions.strip(), expected.strip()) - self.assertEqual(actual_labels.strip(), expected_labels.strip()) - - def test_basic(self): - input = """ - inst(OP, (--)) { - SPAM(); - } - """ - output = """ -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_OP(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - SPAM(); - } - DISPATCH(); -} - """ - output_labels = "" - self.run_cases_test(input, output, output_labels) - - def test_label_transformed(self): - """This tests that the labels and their gotos/DISPATCH get transformed to tail calls in the - tail-calling interpreter, while staying the same/becoming an entry call in the normal interpreter. - """ - input = """ - inst(OP, (--)) { - SPAM(); - } - - label(hello) { - EGGS(); - if (x) { - goto baz; - } - DISPATCH(); - } - """ - output = """ -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_hello(TAIL_CALL_PARAMS) -{ - EGGS(); - if (x) { - TAIL_CALL(baz); - } - DISPATCH(); -} - - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_OP(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - SPAM(); - } - DISPATCH(); - hello: - TAIL_CALL(hello); -} - """ - output_labels = """ -hello: - { - EGGS(); - if (x) { - goto baz; - } - return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0); - } - """ - self.run_cases_test(input, output, output_labels) + self.run_cases_test(input, output) class TestGeneratedAbstractCases(unittest.TestCase): diff --git a/Makefile.pre.in b/Makefile.pre.in index 9aba7881f5973f..67acf0fc520087 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1985,7 +1985,7 @@ Objects/mimalloc/page.o: $(srcdir)/Objects/mimalloc/page-queue.c .PHONY: regen-cases regen-cases: \ regen-opcode-ids regen-opcode-targets regen-uop-ids regen-opcode-metadata-py \ - regen-generated-cases regen-generated-tail-call-handlers regen-executor-cases regen-optimizer-cases \ + regen-generated-cases regen-executor-cases regen-optimizer-cases \ regen-opcode-metadata regen-uop-metadata .PHONY: regen-opcode-ids @@ -2018,12 +2018,6 @@ regen-generated-cases: -o $(srcdir)/Python/generated_cases.c.h.new $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new -.PHONY: regen-generated-tail-call-handlers -regen-generated-tail-call-handlers: - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier1_tail_call_generator.py \ - -o $(srcdir)/Python/generated_tail_call_handlers.c.h.new $(srcdir)/Python/bytecodes.c - $(UPDATE_FILE) $(srcdir)/Python/generated_tail_call_handlers.c.h $(srcdir)/Python/generated_tail_call_handlers.c.h.new - .PHONY: regen-executor-cases regen-executor-cases: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier2_generator.py \ @@ -2061,7 +2055,6 @@ Python/ceval.o: \ $(srcdir)/Python/ceval_macros.h \ $(srcdir)/Python/condvar.h \ $(srcdir)/Python/generated_cases.c.h \ - $(srcdir)/Python/generated_tail_call_handlers.c.h \ $(srcdir)/Python/executor_cases.c.h \ $(srcdir)/Python/opcode_targets.h diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d300cc303890f0..cfd21d3b529317 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1093,7 +1093,7 @@ dummy_func( if (err) { assert(oparg == 0); monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } ERROR_IF(true, error); } @@ -1359,7 +1359,7 @@ dummy_func( assert(exc && PyExceptionInstance_Check(exc)); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } tier1 inst(END_ASYNC_FOR, (awaitable_st, exc_st -- )) { @@ -1374,7 +1374,7 @@ dummy_func( Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } } @@ -1394,7 +1394,7 @@ dummy_func( else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } } @@ -4838,7 +4838,7 @@ dummy_func( tstate, frame, this_instr, prev_instr); if (original_opcode < 0) { next_instr = this_instr+1; - goto error; + JUMP_TO_LABEL(error); } next_instr = frame->instr_ptr; if (next_instr != this_instr) { @@ -5203,22 +5203,22 @@ dummy_func( label(pop_4_error) { STACK_SHRINK(4); - goto error; + JUMP_TO_LABEL(error); } label(pop_3_error) { STACK_SHRINK(3); - goto error; + JUMP_TO_LABEL(error); } label(pop_2_error) { STACK_SHRINK(2); - goto error; + JUMP_TO_LABEL(error); } label(pop_1_error) { STACK_SHRINK(1); - goto error; + JUMP_TO_LABEL(error); } label(error) { @@ -5241,7 +5241,7 @@ dummy_func( } } _PyEval_MonitorRaise(tstate, frame, next_instr-1); - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } label(exception_unwind) { @@ -5260,7 +5260,7 @@ dummy_func( assert(STACK_LEVEL() == 0); _PyFrame_SetStackPointer(frame, stack_pointer); monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; + JUMP_TO_LABEL(exit_unwind); } assert(STACK_LEVEL() >= level); @@ -5272,7 +5272,7 @@ dummy_func( int frame_lasti = _PyInterpreterFrame_LASTI(frame); PyObject *lasti = PyLong_FromLong(frame_lasti); if (lasti == NULL) { - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } PUSH(PyStackRef_FromPyObjectSteal(lasti)); } @@ -5286,13 +5286,17 @@ dummy_func( next_instr = _PyFrame_GetBytecode(frame) + handler; if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } /* Resume normal execution */ #ifdef LLTRACE if (frame->lltrace >= 5) { lltrace_resume_frame(frame); } +#endif +#ifdef Py_TAIL_CALL_INTERP + int opcode; + (void)(opcode); #endif DISPATCH(); } @@ -5314,12 +5318,12 @@ dummy_func( } next_instr = frame->instr_ptr; stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } label(start_frame) { if (_Py_EnterRecursivePy(tstate)) { - goto exit_unwind; + JUMP_TO_LABEL(exit_unwind); } next_instr = frame->instr_ptr; stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5329,7 +5333,7 @@ dummy_func( int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); frame->lltrace = lltrace; if (lltrace < 0) { - goto exit_unwind; + JUMP_TO_LABEL(exit_unwind); } } #endif @@ -5341,6 +5345,10 @@ dummy_func( assert(!_PyErr_Occurred(tstate)); #endif +#ifdef Py_TAIL_CALL_INTERP + int opcode; + (void)(opcode); +#endif DISPATCH(); } diff --git a/Python/ceval.c b/Python/ceval.c index 6ff46c326f5b77..4655421b94ee8f 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -768,14 +768,7 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) #ifdef Py_TAIL_CALL_INTERP -#include "generated_tail_call_handlers.c.h" -static inline PyObject * -_TAIL_CALL_entry(TAIL_CALL_PARAMS) -{ - int opcode = next_instr->op.code; - oparg = next_instr->op.arg; - return (INSTRUCTION_TABLE[opcode])(TAIL_CALL_ARGS); -} +#include "generated_cases.c.h" #endif PyObject* _Py_HOT_FUNCTION @@ -856,7 +849,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int next_instr = frame->instr_ptr; stack_pointer = _PyFrame_GetStackPointer(frame); monitor_throw(tstate, frame, next_instr); +#ifdef Py_TAIL_CALL_INTERP + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, 0); +#else goto error; +#endif } #if defined(_Py_TIER2) && !defined(_Py_JIT) @@ -865,14 +862,19 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int const _PyUOpInstruction *next_uop = NULL; #endif - goto start_frame; - #ifdef Py_TAIL_CALL_INTERP -# include "generated_tail_call_labels.c.h" + return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, 0); #else + goto start_frame; +#endif + +#ifndef Py_TAIL_CALL_INTERP # include "generated_cases.c.h" #endif + + + #ifdef _Py_TIER2 // Tier 2 is also here! diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 60c4b42f3e650d..218becbac29012 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -77,23 +77,34 @@ // Note: [[clang::musttail]] works for GCC 15, but not __attribute__((musttail)) at the moment. # define Py_MUSTTAIL [[clang::musttail]] # define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) - Py_PRESERVE_NONE_CC - typedef PyObject* (*py_tail_call_funcptr)(TAIL_CALL_PARAMS); -# define DISPATCH_GOTO() do { \ - Py_MUSTTAIL \ - return (INSTRUCTION_TABLE[opcode])(TAIL_CALL_ARGS); \ -} while (0) -# define TAIL_CALL(name) do { \ - Py_MUSTTAIL \ - return (_TAIL_CALL_##name)(TAIL_CALL_ARGS); \ -} while (0) - + Py_PRESERVE_NONE_CC typedef PyObject* (*py_tail_call_funcptr)(TAIL_CALL_PARAMS); + +# define TARGET(op) PyObject *_TAIL_CALL_##op(TAIL_CALL_PARAMS) +# define DISPATCH_GOTO() \ + do { \ + Py_MUSTTAIL return (INSTRUCTION_TABLE[opcode])(TAIL_CALL_ARGS); \ + } while (0) +# define JUMP_TO_LABEL(name) \ + do { \ + Py_MUSTTAIL return (_TAIL_CALL_##name)(TAIL_CALL_ARGS); \ + } while (0) +# define JUMP_TO_PREDICTED(name) \ + do { \ + Py_MUSTTAIL return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, this_instr, oparg); \ + } while (0) +# define LABEL(name) TARGET(name) #elif USE_COMPUTED_GOTOS # define TARGET(op) TARGET_##op: # define DISPATCH_GOTO() goto *opcode_targets[opcode] +# define JUMP_TO_LABEL(name) goto name; +# define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; +# define LABEL(name) name: #else # define TARGET(op) case op: TARGET_##op: # define DISPATCH_GOTO() goto dispatch_opcode +# define JUMP_TO_LABEL(name) goto name; +# define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; +# define LABEL(name) name: #endif /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ @@ -110,7 +121,7 @@ do { \ int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); \ frame->lltrace = lltrace; \ if (lltrace < 0) { \ - goto exit_unwind; \ + JUMP_TO_LABEL(exit_unwind); \ } \ } while (0) #else @@ -125,15 +136,8 @@ do { \ /* Do interpreter dispatch accounting for tracing and instrumentation */ -#ifdef Py_TAIL_CALL_INTERP -# define DEFINE_OPCODE_IF_NEEDED() int opcode; -#else -# define DEFINE_OPCODE_IF_NEEDED() -#endif - #define DISPATCH() \ { \ - DEFINE_OPCODE_IF_NEEDED(); \ assert(frame->stackpointer == NULL); \ NEXTOPARG(); \ PRE_DISPATCH_GOTO(); \ @@ -154,11 +158,11 @@ do { \ assert((NEW_FRAME)->previous == frame); \ frame = tstate->current_frame = (NEW_FRAME); \ CALL_STAT_INC(inlined_py_calls); \ - goto start_frame; \ + JUMP_TO_LABEL(start_frame); \ } while (0) // Use this instead of 'goto error' so Tier 2 can go to a different label -#define GOTO_ERROR(LABEL) goto LABEL +#define GOTO_ERROR(LABEL) JUMP_TO_LABEL(LABEL) /* Tuple access macros */ @@ -354,7 +358,7 @@ do { \ stack_pointer = _PyFrame_GetStackPointer(frame); \ if (next_instr == NULL) { \ next_instr = (dest)+1; \ - goto error; \ + JUMP_TO_LABEL(error); \ } \ } \ } while (0); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b332970f27daf5..ee4e09f449fab4 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7,18 +7,527 @@ #error "This file is for Tier 1 only" #endif #define TIER_ONE 1 +#ifdef Py_TAIL_CALL_INTERP +static py_tail_call_funcptr INSTRUCTION_TABLE[256]; + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_4_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_3_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_MORTAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { + int opcode = next_instr->op.code; + _PyErr_Format(tstate, PyExc_SystemError, + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); + JUMP_TO_LABEL(error); +} + +static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { + [BINARY_OP] = _TAIL_CALL_BINARY_OP, + [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, + [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, + [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_BINARY_OP_ADD_UNICODE, + [BINARY_OP_EXTEND] = _TAIL_CALL_BINARY_OP_EXTEND, + [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE, + [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT, + [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_BINARY_OP_MULTIPLY_INT, + [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT, + [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_BINARY_OP_SUBTRACT_INT, + [BINARY_SLICE] = _TAIL_CALL_BINARY_SLICE, + [BINARY_SUBSCR] = _TAIL_CALL_BINARY_SUBSCR, + [BINARY_SUBSCR_DICT] = _TAIL_CALL_BINARY_SUBSCR_DICT, + [BINARY_SUBSCR_GETITEM] = _TAIL_CALL_BINARY_SUBSCR_GETITEM, + [BINARY_SUBSCR_LIST_INT] = _TAIL_CALL_BINARY_SUBSCR_LIST_INT, + [BINARY_SUBSCR_STR_INT] = _TAIL_CALL_BINARY_SUBSCR_STR_INT, + [BINARY_SUBSCR_TUPLE_INT] = _TAIL_CALL_BINARY_SUBSCR_TUPLE_INT, + [BUILD_LIST] = _TAIL_CALL_BUILD_LIST, + [BUILD_MAP] = _TAIL_CALL_BUILD_MAP, + [BUILD_SET] = _TAIL_CALL_BUILD_SET, + [BUILD_SLICE] = _TAIL_CALL_BUILD_SLICE, + [BUILD_STRING] = _TAIL_CALL_BUILD_STRING, + [BUILD_TUPLE] = _TAIL_CALL_BUILD_TUPLE, + [CACHE] = _TAIL_CALL_CACHE, + [CALL] = _TAIL_CALL_CALL, + [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT, + [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS, + [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_CALL_BOUND_METHOD_GENERAL, + [CALL_BUILTIN_CLASS] = _TAIL_CALL_CALL_BUILTIN_CLASS, + [CALL_BUILTIN_FAST] = _TAIL_CALL_CALL_BUILTIN_FAST, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS, + [CALL_BUILTIN_O] = _TAIL_CALL_CALL_BUILTIN_O, + [CALL_FUNCTION_EX] = _TAIL_CALL_CALL_FUNCTION_EX, + [CALL_INTRINSIC_1] = _TAIL_CALL_CALL_INTRINSIC_1, + [CALL_INTRINSIC_2] = _TAIL_CALL_CALL_INTRINSIC_2, + [CALL_ISINSTANCE] = _TAIL_CALL_CALL_ISINSTANCE, + [CALL_KW] = _TAIL_CALL_CALL_KW, + [CALL_KW_BOUND_METHOD] = _TAIL_CALL_CALL_KW_BOUND_METHOD, + [CALL_KW_NON_PY] = _TAIL_CALL_CALL_KW_NON_PY, + [CALL_KW_PY] = _TAIL_CALL_CALL_KW_PY, + [CALL_LEN] = _TAIL_CALL_CALL_LEN, + [CALL_LIST_APPEND] = _TAIL_CALL_CALL_LIST_APPEND, + [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS, + [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O, + [CALL_NON_PY_GENERAL] = _TAIL_CALL_CALL_NON_PY_GENERAL, + [CALL_PY_EXACT_ARGS] = _TAIL_CALL_CALL_PY_EXACT_ARGS, + [CALL_PY_GENERAL] = _TAIL_CALL_CALL_PY_GENERAL, + [CALL_STR_1] = _TAIL_CALL_CALL_STR_1, + [CALL_TUPLE_1] = _TAIL_CALL_CALL_TUPLE_1, + [CALL_TYPE_1] = _TAIL_CALL_CALL_TYPE_1, + [CHECK_EG_MATCH] = _TAIL_CALL_CHECK_EG_MATCH, + [CHECK_EXC_MATCH] = _TAIL_CALL_CHECK_EXC_MATCH, + [CLEANUP_THROW] = _TAIL_CALL_CLEANUP_THROW, + [COMPARE_OP] = _TAIL_CALL_COMPARE_OP, + [COMPARE_OP_FLOAT] = _TAIL_CALL_COMPARE_OP_FLOAT, + [COMPARE_OP_INT] = _TAIL_CALL_COMPARE_OP_INT, + [COMPARE_OP_STR] = _TAIL_CALL_COMPARE_OP_STR, + [CONTAINS_OP] = _TAIL_CALL_CONTAINS_OP, + [CONTAINS_OP_DICT] = _TAIL_CALL_CONTAINS_OP_DICT, + [CONTAINS_OP_SET] = _TAIL_CALL_CONTAINS_OP_SET, + [CONVERT_VALUE] = _TAIL_CALL_CONVERT_VALUE, + [COPY] = _TAIL_CALL_COPY, + [COPY_FREE_VARS] = _TAIL_CALL_COPY_FREE_VARS, + [DELETE_ATTR] = _TAIL_CALL_DELETE_ATTR, + [DELETE_DEREF] = _TAIL_CALL_DELETE_DEREF, + [DELETE_FAST] = _TAIL_CALL_DELETE_FAST, + [DELETE_GLOBAL] = _TAIL_CALL_DELETE_GLOBAL, + [DELETE_NAME] = _TAIL_CALL_DELETE_NAME, + [DELETE_SUBSCR] = _TAIL_CALL_DELETE_SUBSCR, + [DICT_MERGE] = _TAIL_CALL_DICT_MERGE, + [DICT_UPDATE] = _TAIL_CALL_DICT_UPDATE, + [END_ASYNC_FOR] = _TAIL_CALL_END_ASYNC_FOR, + [END_FOR] = _TAIL_CALL_END_FOR, + [END_SEND] = _TAIL_CALL_END_SEND, + [ENTER_EXECUTOR] = _TAIL_CALL_ENTER_EXECUTOR, + [EXIT_INIT_CHECK] = _TAIL_CALL_EXIT_INIT_CHECK, + [EXTENDED_ARG] = _TAIL_CALL_EXTENDED_ARG, + [FORMAT_SIMPLE] = _TAIL_CALL_FORMAT_SIMPLE, + [FORMAT_WITH_SPEC] = _TAIL_CALL_FORMAT_WITH_SPEC, + [FOR_ITER] = _TAIL_CALL_FOR_ITER, + [FOR_ITER_GEN] = _TAIL_CALL_FOR_ITER_GEN, + [FOR_ITER_LIST] = _TAIL_CALL_FOR_ITER_LIST, + [FOR_ITER_RANGE] = _TAIL_CALL_FOR_ITER_RANGE, + [FOR_ITER_TUPLE] = _TAIL_CALL_FOR_ITER_TUPLE, + [GET_AITER] = _TAIL_CALL_GET_AITER, + [GET_ANEXT] = _TAIL_CALL_GET_ANEXT, + [GET_AWAITABLE] = _TAIL_CALL_GET_AWAITABLE, + [GET_ITER] = _TAIL_CALL_GET_ITER, + [GET_LEN] = _TAIL_CALL_GET_LEN, + [GET_YIELD_FROM_ITER] = _TAIL_CALL_GET_YIELD_FROM_ITER, + [IMPORT_FROM] = _TAIL_CALL_IMPORT_FROM, + [IMPORT_NAME] = _TAIL_CALL_IMPORT_NAME, + [INSTRUMENTED_CALL] = _TAIL_CALL_INSTRUMENTED_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_CALL_KW] = _TAIL_CALL_INSTRUMENTED_CALL_KW, + [INSTRUMENTED_END_FOR] = _TAIL_CALL_INSTRUMENTED_END_FOR, + [INSTRUMENTED_END_SEND] = _TAIL_CALL_INSTRUMENTED_END_SEND, + [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_INSTRUMENTED_FOR_ITER, + [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_INSTRUMENTED_INSTRUCTION, + [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD, + [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD, + [INSTRUMENTED_LINE] = _TAIL_CALL_INSTRUMENTED_LINE, + [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR, + [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_INSTRUMENTED_NOT_TAKEN, + [INSTRUMENTED_POP_ITER] = _TAIL_CALL_INSTRUMENTED_POP_ITER, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE, + [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE, + [INSTRUMENTED_RESUME] = _TAIL_CALL_INSTRUMENTED_RESUME, + [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_INSTRUMENTED_RETURN_VALUE, + [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_INSTRUMENTED_YIELD_VALUE, + [INTERPRETER_EXIT] = _TAIL_CALL_INTERPRETER_EXIT, + [IS_OP] = _TAIL_CALL_IS_OP, + [JUMP_BACKWARD] = _TAIL_CALL_JUMP_BACKWARD, + [JUMP_BACKWARD_JIT] = _TAIL_CALL_JUMP_BACKWARD_JIT, + [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT, + [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_JUMP_BACKWARD_NO_JIT, + [JUMP_FORWARD] = _TAIL_CALL_JUMP_FORWARD, + [LIST_APPEND] = _TAIL_CALL_LIST_APPEND, + [LIST_EXTEND] = _TAIL_CALL_LIST_EXTEND, + [LOAD_ATTR] = _TAIL_CALL_LOAD_ATTR, + [LOAD_ATTR_CLASS] = _TAIL_CALL_LOAD_ATTR_CLASS, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, + [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE, + [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT, + [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT, + [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES, + [LOAD_ATTR_MODULE] = _TAIL_CALL_LOAD_ATTR_MODULE, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, + [LOAD_ATTR_PROPERTY] = _TAIL_CALL_LOAD_ATTR_PROPERTY, + [LOAD_ATTR_SLOT] = _TAIL_CALL_LOAD_ATTR_SLOT, + [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_LOAD_ATTR_WITH_HINT, + [LOAD_BUILD_CLASS] = _TAIL_CALL_LOAD_BUILD_CLASS, + [LOAD_COMMON_CONSTANT] = _TAIL_CALL_LOAD_COMMON_CONSTANT, + [LOAD_CONST] = _TAIL_CALL_LOAD_CONST, + [LOAD_CONST_IMMORTAL] = _TAIL_CALL_LOAD_CONST_IMMORTAL, + [LOAD_CONST_MORTAL] = _TAIL_CALL_LOAD_CONST_MORTAL, + [LOAD_DEREF] = _TAIL_CALL_LOAD_DEREF, + [LOAD_FAST] = _TAIL_CALL_LOAD_FAST, + [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_LOAD_FAST_AND_CLEAR, + [LOAD_FAST_CHECK] = _TAIL_CALL_LOAD_FAST_CHECK, + [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_LOAD_FAST_LOAD_FAST, + [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF, + [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS, + [LOAD_GLOBAL] = _TAIL_CALL_LOAD_GLOBAL, + [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_LOAD_GLOBAL_BUILTIN, + [LOAD_GLOBAL_MODULE] = _TAIL_CALL_LOAD_GLOBAL_MODULE, + [LOAD_LOCALS] = _TAIL_CALL_LOAD_LOCALS, + [LOAD_NAME] = _TAIL_CALL_LOAD_NAME, + [LOAD_SMALL_INT] = _TAIL_CALL_LOAD_SMALL_INT, + [LOAD_SPECIAL] = _TAIL_CALL_LOAD_SPECIAL, + [LOAD_SUPER_ATTR] = _TAIL_CALL_LOAD_SUPER_ATTR, + [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_LOAD_SUPER_ATTR_ATTR, + [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_LOAD_SUPER_ATTR_METHOD, + [MAKE_CELL] = _TAIL_CALL_MAKE_CELL, + [MAKE_FUNCTION] = _TAIL_CALL_MAKE_FUNCTION, + [MAP_ADD] = _TAIL_CALL_MAP_ADD, + [MATCH_CLASS] = _TAIL_CALL_MATCH_CLASS, + [MATCH_KEYS] = _TAIL_CALL_MATCH_KEYS, + [MATCH_MAPPING] = _TAIL_CALL_MATCH_MAPPING, + [MATCH_SEQUENCE] = _TAIL_CALL_MATCH_SEQUENCE, + [NOP] = _TAIL_CALL_NOP, + [NOT_TAKEN] = _TAIL_CALL_NOT_TAKEN, + [POP_EXCEPT] = _TAIL_CALL_POP_EXCEPT, + [POP_ITER] = _TAIL_CALL_POP_ITER, + [POP_JUMP_IF_FALSE] = _TAIL_CALL_POP_JUMP_IF_FALSE, + [POP_JUMP_IF_NONE] = _TAIL_CALL_POP_JUMP_IF_NONE, + [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_POP_JUMP_IF_NOT_NONE, + [POP_JUMP_IF_TRUE] = _TAIL_CALL_POP_JUMP_IF_TRUE, + [POP_TOP] = _TAIL_CALL_POP_TOP, + [PUSH_EXC_INFO] = _TAIL_CALL_PUSH_EXC_INFO, + [PUSH_NULL] = _TAIL_CALL_PUSH_NULL, + [RAISE_VARARGS] = _TAIL_CALL_RAISE_VARARGS, + [RERAISE] = _TAIL_CALL_RERAISE, + [RESERVED] = _TAIL_CALL_RESERVED, + [RESUME] = _TAIL_CALL_RESUME, + [RESUME_CHECK] = _TAIL_CALL_RESUME_CHECK, + [RETURN_GENERATOR] = _TAIL_CALL_RETURN_GENERATOR, + [RETURN_VALUE] = _TAIL_CALL_RETURN_VALUE, + [SEND] = _TAIL_CALL_SEND, + [SEND_GEN] = _TAIL_CALL_SEND_GEN, + [SETUP_ANNOTATIONS] = _TAIL_CALL_SETUP_ANNOTATIONS, + [SET_ADD] = _TAIL_CALL_SET_ADD, + [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_SET_FUNCTION_ATTRIBUTE, + [SET_UPDATE] = _TAIL_CALL_SET_UPDATE, + [STORE_ATTR] = _TAIL_CALL_STORE_ATTR, + [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE, + [STORE_ATTR_SLOT] = _TAIL_CALL_STORE_ATTR_SLOT, + [STORE_ATTR_WITH_HINT] = _TAIL_CALL_STORE_ATTR_WITH_HINT, + [STORE_DEREF] = _TAIL_CALL_STORE_DEREF, + [STORE_FAST] = _TAIL_CALL_STORE_FAST, + [STORE_FAST_LOAD_FAST] = _TAIL_CALL_STORE_FAST_LOAD_FAST, + [STORE_FAST_STORE_FAST] = _TAIL_CALL_STORE_FAST_STORE_FAST, + [STORE_GLOBAL] = _TAIL_CALL_STORE_GLOBAL, + [STORE_NAME] = _TAIL_CALL_STORE_NAME, + [STORE_SLICE] = _TAIL_CALL_STORE_SLICE, + [STORE_SUBSCR] = _TAIL_CALL_STORE_SUBSCR, + [STORE_SUBSCR_DICT] = _TAIL_CALL_STORE_SUBSCR_DICT, + [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_STORE_SUBSCR_LIST_INT, + [SWAP] = _TAIL_CALL_SWAP, + [TO_BOOL] = _TAIL_CALL_TO_BOOL, + [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TO_BOOL_ALWAYS_TRUE, + [TO_BOOL_BOOL] = _TAIL_CALL_TO_BOOL_BOOL, + [TO_BOOL_INT] = _TAIL_CALL_TO_BOOL_INT, + [TO_BOOL_LIST] = _TAIL_CALL_TO_BOOL_LIST, + [TO_BOOL_NONE] = _TAIL_CALL_TO_BOOL_NONE, + [TO_BOOL_STR] = _TAIL_CALL_TO_BOOL_STR, + [UNARY_INVERT] = _TAIL_CALL_UNARY_INVERT, + [UNARY_NEGATIVE] = _TAIL_CALL_UNARY_NEGATIVE, + [UNARY_NOT] = _TAIL_CALL_UNARY_NOT, + [UNPACK_EX] = _TAIL_CALL_UNPACK_EX, + [UNPACK_SEQUENCE] = _TAIL_CALL_UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_UNPACK_SEQUENCE_LIST, + [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TUPLE, + [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, + [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, + [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, + [118] = _TAIL_CALL_UNKNOWN_OPCODE, + [119] = _TAIL_CALL_UNKNOWN_OPCODE, + [120] = _TAIL_CALL_UNKNOWN_OPCODE, + [121] = _TAIL_CALL_UNKNOWN_OPCODE, + [122] = _TAIL_CALL_UNKNOWN_OPCODE, + [123] = _TAIL_CALL_UNKNOWN_OPCODE, + [124] = _TAIL_CALL_UNKNOWN_OPCODE, + [125] = _TAIL_CALL_UNKNOWN_OPCODE, + [126] = _TAIL_CALL_UNKNOWN_OPCODE, + [127] = _TAIL_CALL_UNKNOWN_OPCODE, + [128] = _TAIL_CALL_UNKNOWN_OPCODE, + [129] = _TAIL_CALL_UNKNOWN_OPCODE, + [130] = _TAIL_CALL_UNKNOWN_OPCODE, + [131] = _TAIL_CALL_UNKNOWN_OPCODE, + [132] = _TAIL_CALL_UNKNOWN_OPCODE, + [133] = _TAIL_CALL_UNKNOWN_OPCODE, + [134] = _TAIL_CALL_UNKNOWN_OPCODE, + [135] = _TAIL_CALL_UNKNOWN_OPCODE, + [136] = _TAIL_CALL_UNKNOWN_OPCODE, + [137] = _TAIL_CALL_UNKNOWN_OPCODE, + [138] = _TAIL_CALL_UNKNOWN_OPCODE, + [139] = _TAIL_CALL_UNKNOWN_OPCODE, + [140] = _TAIL_CALL_UNKNOWN_OPCODE, + [141] = _TAIL_CALL_UNKNOWN_OPCODE, + [142] = _TAIL_CALL_UNKNOWN_OPCODE, + [143] = _TAIL_CALL_UNKNOWN_OPCODE, + [144] = _TAIL_CALL_UNKNOWN_OPCODE, + [145] = _TAIL_CALL_UNKNOWN_OPCODE, + [146] = _TAIL_CALL_UNKNOWN_OPCODE, + [147] = _TAIL_CALL_UNKNOWN_OPCODE, + [148] = _TAIL_CALL_UNKNOWN_OPCODE, + [232] = _TAIL_CALL_UNKNOWN_OPCODE, + [233] = _TAIL_CALL_UNKNOWN_OPCODE, + [234] = _TAIL_CALL_UNKNOWN_OPCODE, +}; +#endif /* Py_TAIL_CALL_INTERP */ + #ifndef Py_TAIL_CALL_INTERP - #if !USE_COMPUTED_GOTOS dispatch_opcode: switch (opcode) #endif { +#endif /* Py_TAIL_CALL_INTERP */ /* BEGIN INSTRUCTIONS */ TARGET(BINARY_OP) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP); @@ -60,7 +569,7 @@ PyStackRef_CLOSE(lhs); PyStackRef_CLOSE(rhs); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -71,6 +580,14 @@ } TARGET(BINARY_OP_ADD_FLOAT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); @@ -87,12 +604,12 @@ if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - goto PREDICTED_BINARY_OP; + JUMP_TO_PREDICTED(BINARY_OP); } if (!PyFloat_CheckExact(right_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - goto PREDICTED_BINARY_OP; + JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip 5 cache entries */ @@ -108,7 +625,7 @@ ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -119,6 +636,14 @@ } TARGET(BINARY_OP_ADD_INT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_INT); @@ -135,12 +660,12 @@ if (!PyLong_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - goto PREDICTED_BINARY_OP; + JUMP_TO_PREDICTED(BINARY_OP); } if (!PyLong_CheckExact(right_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - goto PREDICTED_BINARY_OP; + JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip 5 cache entries */ @@ -155,7 +680,7 @@ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -166,6 +691,14 @@ } TARGET(BINARY_OP_ADD_UNICODE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); @@ -182,12 +715,12 @@ if (!PyUnicode_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - goto PREDICTED_BINARY_OP; + JUMP_TO_PREDICTED(BINARY_OP); } if (!PyUnicode_CheckExact(right_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - goto PREDICTED_BINARY_OP; + JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip 5 cache entries */ @@ -202,7 +735,7 @@ PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -213,8 +746,13 @@ } TARGET(BINARY_OP_EXTEND) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_EXTEND); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -238,7 +776,7 @@ if (!res) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - goto PREDICTED_BINARY_OP; + JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip -4 cache entry */ @@ -264,6 +802,14 @@ } TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); @@ -279,12 +825,12 @@ if (!PyUnicode_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - goto PREDICTED_BINARY_OP; + JUMP_TO_PREDICTED(BINARY_OP); } if (!PyUnicode_CheckExact(right_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - goto PREDICTED_BINARY_OP; + JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip 5 cache entries */ @@ -305,7 +851,7 @@ if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - goto PREDICTED_BINARY_OP; + JUMP_TO_PREDICTED(BINARY_OP); } STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. @@ -326,7 +872,7 @@ *target_local = PyStackRef_FromPyObjectSteal(temp); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (PyStackRef_IsNull(*target_local)) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } #if TIER_ONE // The STORE_FAST is already done. This is done here in tier one, @@ -341,6 +887,14 @@ } TARGET(BINARY_OP_MULTIPLY_FLOAT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); @@ -357,12 +911,12 @@ if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - goto PREDICTED_BINARY_OP; + JUMP_TO_PREDICTED(BINARY_OP); } if (!PyFloat_CheckExact(right_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - goto PREDICTED_BINARY_OP; + JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip 5 cache entries */ @@ -378,7 +932,7 @@ ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -389,6 +943,14 @@ } TARGET(BINARY_OP_MULTIPLY_INT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); @@ -405,12 +967,12 @@ if (!PyLong_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - goto PREDICTED_BINARY_OP; + JUMP_TO_PREDICTED(BINARY_OP); } if (!PyLong_CheckExact(right_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - goto PREDICTED_BINARY_OP; + JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip 5 cache entries */ @@ -425,7 +987,7 @@ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -436,6 +998,14 @@ } TARGET(BINARY_OP_SUBTRACT_FLOAT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); @@ -452,12 +1022,12 @@ if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - goto PREDICTED_BINARY_OP; + JUMP_TO_PREDICTED(BINARY_OP); } if (!PyFloat_CheckExact(right_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - goto PREDICTED_BINARY_OP; + JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip 5 cache entries */ @@ -473,7 +1043,7 @@ ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -484,6 +1054,14 @@ } TARGET(BINARY_OP_SUBTRACT_INT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); @@ -500,12 +1078,12 @@ if (!PyLong_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - goto PREDICTED_BINARY_OP; + JUMP_TO_PREDICTED(BINARY_OP); } if (!PyLong_CheckExact(right_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - goto PREDICTED_BINARY_OP; + JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip 5 cache entries */ @@ -520,7 +1098,7 @@ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -531,6 +1109,10 @@ } TARGET(BINARY_SLICE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BINARY_SLICE); @@ -576,7 +1158,7 @@ PyStackRef_CLOSE(container); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -587,6 +1169,10 @@ } TARGET(BINARY_SUBSCR) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR); @@ -625,7 +1211,7 @@ PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -636,6 +1222,14 @@ } TARGET(BINARY_SUBSCR_DICT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_DICT); @@ -651,7 +1245,7 @@ if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - goto PREDICTED_BINARY_SUBSCR; + JUMP_TO_PREDICTED(BINARY_SUBSCR); } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o; @@ -666,7 +1260,7 @@ PyStackRef_CLOSE(dict_st); PyStackRef_CLOSE(sub_st); if (rc <= 0) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } // not found or error res = PyStackRef_FromPyObjectSteal(res_o); @@ -677,6 +1271,14 @@ } TARGET(BINARY_SUBSCR_GETITEM) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); @@ -691,7 +1293,7 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - goto PREDICTED_BINARY_SUBSCR; + JUMP_TO_PREDICTED(BINARY_SUBSCR); } } // _BINARY_SUBSCR_CHECK_FUNC @@ -701,28 +1303,28 @@ if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - goto PREDICTED_BINARY_SUBSCR; + JUMP_TO_PREDICTED(BINARY_SUBSCR); } PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); if (getitem_o == NULL) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - goto PREDICTED_BINARY_SUBSCR; + JUMP_TO_PREDICTED(BINARY_SUBSCR); } assert(PyFunction_Check(getitem_o)); uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - goto PREDICTED_BINARY_SUBSCR; + JUMP_TO_PREDICTED(BINARY_SUBSCR); } PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - goto PREDICTED_BINARY_SUBSCR; + JUMP_TO_PREDICTED(BINARY_SUBSCR); } getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); @@ -756,6 +1358,14 @@ } TARGET(BINARY_SUBSCR_LIST_INT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); @@ -771,18 +1381,18 @@ if (!PyLong_CheckExact(sub)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - goto PREDICTED_BINARY_SUBSCR; + JUMP_TO_PREDICTED(BINARY_SUBSCR); } if (!PyList_CheckExact(list)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - goto PREDICTED_BINARY_SUBSCR; + JUMP_TO_PREDICTED(BINARY_SUBSCR); } // Deopt unless 0 <= sub < PyList_Size(list) if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - goto PREDICTED_BINARY_SUBSCR; + JUMP_TO_PREDICTED(BINARY_SUBSCR); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; #ifdef Py_GIL_DISABLED @@ -792,14 +1402,14 @@ if (res_o == NULL) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - goto PREDICTED_BINARY_SUBSCR; + JUMP_TO_PREDICTED(BINARY_SUBSCR); } STAT_INC(BINARY_SUBSCR, hit); #else if (index >= PyList_GET_SIZE(list)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - goto PREDICTED_BINARY_SUBSCR; + JUMP_TO_PREDICTED(BINARY_SUBSCR); } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyList_GET_ITEM(list, index); @@ -820,6 +1430,14 @@ } TARGET(BINARY_SUBSCR_STR_INT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); @@ -835,30 +1453,30 @@ if (!PyLong_CheckExact(sub)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - goto PREDICTED_BINARY_SUBSCR; + JUMP_TO_PREDICTED(BINARY_SUBSCR); } if (!PyUnicode_CheckExact(str)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - goto PREDICTED_BINARY_SUBSCR; + JUMP_TO_PREDICTED(BINARY_SUBSCR); } if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - goto PREDICTED_BINARY_SUBSCR; + JUMP_TO_PREDICTED(BINARY_SUBSCR); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (PyUnicode_GET_LENGTH(str) <= index) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - goto PREDICTED_BINARY_SUBSCR; + JUMP_TO_PREDICTED(BINARY_SUBSCR); } // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - goto PREDICTED_BINARY_SUBSCR; + JUMP_TO_PREDICTED(BINARY_SUBSCR); } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; @@ -876,6 +1494,14 @@ } TARGET(BINARY_SUBSCR_TUPLE_INT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); @@ -891,24 +1517,24 @@ if (!PyLong_CheckExact(sub)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - goto PREDICTED_BINARY_SUBSCR; + JUMP_TO_PREDICTED(BINARY_SUBSCR); } if (!PyTuple_CheckExact(tuple)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - goto PREDICTED_BINARY_SUBSCR; + JUMP_TO_PREDICTED(BINARY_SUBSCR); } // Deopt unless 0 <= sub < PyTuple_Size(list) if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - goto PREDICTED_BINARY_SUBSCR; + JUMP_TO_PREDICTED(BINARY_SUBSCR); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (index >= PyTuple_GET_SIZE(tuple)) { UPDATE_MISS_STATS(BINARY_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - goto PREDICTED_BINARY_SUBSCR; + JUMP_TO_PREDICTED(BINARY_SUBSCR); } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyTuple_GET_ITEM(tuple, index); @@ -928,6 +1554,10 @@ } TARGET(BUILD_LIST) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_LIST); @@ -936,7 +1566,7 @@ values = &stack_pointer[-oparg]; PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); if (list_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } list = PyStackRef_FromPyObjectSteal(list_o); stack_pointer[-oparg] = list; @@ -946,6 +1576,10 @@ } TARGET(BUILD_MAP) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_MAP); @@ -959,7 +1593,7 @@ } stack_pointer += -oparg*2; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *map_o = _PyDict_FromItems( @@ -974,7 +1608,7 @@ if (map_o == NULL) { stack_pointer += -oparg*2; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } map = PyStackRef_FromPyObjectSteal(map_o); stack_pointer[-oparg*2] = map; @@ -984,6 +1618,10 @@ } TARGET(BUILD_SET) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SET); @@ -999,7 +1637,7 @@ } stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } int err = 0; for (int i = 0; i < oparg; i++) { @@ -1018,7 +1656,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(set_o); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } set = PyStackRef_FromPyObjectSteal(set_o); stack_pointer[-oparg] = set; @@ -1028,6 +1666,10 @@ } TARGET(BUILD_SLICE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SLICE); @@ -1044,7 +1686,7 @@ if (slice_o == NULL) { stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } slice = PyStackRef_FromPyObjectSteal(slice_o); stack_pointer[-oparg] = slice; @@ -1054,6 +1696,10 @@ } TARGET(BUILD_STRING) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_STRING); @@ -1067,7 +1713,7 @@ } stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); @@ -1077,7 +1723,7 @@ if (str_o == NULL) { stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } str = PyStackRef_FromPyObjectSteal(str_o); stack_pointer[-oparg] = str; @@ -1087,6 +1733,10 @@ } TARGET(BUILD_TUPLE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_TUPLE); @@ -1095,7 +1745,7 @@ values = &stack_pointer[-oparg]; PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); if (tup_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } tup = PyStackRef_FromPyObjectSteal(tup_o); stack_pointer[-oparg] = tup; @@ -1105,6 +1755,10 @@ } TARGET(CACHE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CACHE); @@ -1114,6 +1768,10 @@ } TARGET(CALL) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL); @@ -1195,7 +1853,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - goto error; + JUMP_TO_LABEL(error); } frame->return_offset = 4 ; DISPATCH_INLINED(new_frame); @@ -1210,7 +1868,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( @@ -1249,7 +1907,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1265,7 +1923,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1278,8 +1936,13 @@ } TARGET(CALL_ALLOC_AND_ENTER_INIT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -1296,7 +1959,7 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } } // _CHECK_AND_ALLOCATE_OBJECT @@ -1311,18 +1974,18 @@ if (!PyStackRef_IsNull(null[0])) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } if (!PyType_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } PyTypeObject *tp = (PyTypeObject *)callable_o; if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } assert(tp->tp_new == PyBaseObject_Type.tp_new); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); @@ -1333,14 +1996,14 @@ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *self_o = PyType_GenericAlloc(tp, 0); stack_pointer = _PyFrame_GetStackPointer(frame); if (self_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } self[0] = PyStackRef_FromPyObjectSteal(self_o); _PyStackRef temp = callable[0]; @@ -1370,7 +2033,7 @@ assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { _PyEval_FrameClearAndPop(tstate, shim); - goto error; + JUMP_TO_LABEL(error); } init_frame = temp; frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; @@ -1399,8 +2062,13 @@ } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -1415,7 +2083,7 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS @@ -1425,12 +2093,12 @@ if (!PyStackRef_IsNull(null[0])) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS @@ -1454,13 +2122,13 @@ if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } } // _CHECK_FUNCTION_EXACT_ARGS @@ -1472,7 +2140,7 @@ if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } } // _CHECK_STACK_SPACE @@ -1483,12 +2151,12 @@ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } } // _INIT_CALL_PY_EXACT_ARGS @@ -1533,8 +2201,13 @@ } TARGET(CALL_BOUND_METHOD_GENERAL) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -1549,7 +2222,7 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } } // _CHECK_METHOD_VERSION @@ -1561,23 +2234,23 @@ if (Py_TYPE(callable_o) != &PyMethod_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } PyObject *func = ((PyMethodObject *)callable_o)->im_func; if (!PyFunction_Check(func)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } if (((PyFunctionObject *)func)->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } if (!PyStackRef_IsNull(null[0])) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } } // _EXPAND_METHOD @@ -1618,7 +2291,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - goto error; + JUMP_TO_LABEL(error); } new_frame = temp; } @@ -1650,6 +2323,14 @@ } TARGET(CALL_BUILTIN_CLASS) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_CLASS); @@ -1669,7 +2350,7 @@ if (!PyType_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } PyTypeObject *tp = (PyTypeObject *)callable_o; int total_args = oparg; @@ -1681,7 +2362,7 @@ if (tp->tp_vectorcall == NULL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -1693,7 +2374,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); @@ -1707,7 +2388,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1723,7 +2404,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1736,6 +2417,14 @@ } TARGET(CALL_BUILTIN_FAST) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST); @@ -1762,12 +2451,12 @@ if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); @@ -1781,7 +2470,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( @@ -1799,7 +2488,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1815,7 +2504,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1828,6 +2517,14 @@ } TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); @@ -1854,12 +2551,12 @@ if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); /* res = func(self, arguments, nargs, kwnames) */ @@ -1877,7 +2574,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); @@ -1892,7 +2589,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1908,7 +2605,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1921,6 +2618,14 @@ } TARGET(CALL_BUILTIN_O) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_O); @@ -1946,23 +2651,23 @@ if (total_args != 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } // CPython promises to check all non-vectorcall function calls. if (tstate->c_recursion_remaining <= 0) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); @@ -1982,7 +2687,7 @@ PyStackRef_CLOSE(callable[0]); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1998,7 +2703,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2011,8 +2716,13 @@ } TARGET(CALL_FUNCTION_EX) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_FUNCTION_EX); opcode = CALL_FUNCTION_EX; @@ -2041,13 +2751,13 @@ int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *tuple_o = PySequence_Tuple(callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (tuple_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } kwargs_out = kwargs_in; stack_pointer += -2; @@ -2088,7 +2798,7 @@ frame, this_instr, func, arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); result_o = PyObject_Call(func, callargs, kwargs); @@ -2135,7 +2845,7 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { - goto error; + JUMP_TO_LABEL(error); } assert( 1 == 1); frame->return_offset = 1; @@ -2167,7 +2877,7 @@ PyStackRef_CLOSE(func_st); stack_pointer = _PyFrame_GetStackPointer(frame); if (result_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } result = PyStackRef_FromPyObjectSteal(result_o); } @@ -2183,7 +2893,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2196,6 +2906,10 @@ } TARGET(CALL_INTRINSIC_1) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_1); @@ -2208,7 +2922,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (res_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; @@ -2216,6 +2930,10 @@ } TARGET(CALL_INTRINSIC_2) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_2); @@ -2233,7 +2951,7 @@ PyStackRef_CLOSE(value2_st); PyStackRef_CLOSE(value1_st); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -2243,6 +2961,14 @@ } TARGET(CALL_ISINSTANCE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ISINSTANCE); @@ -2267,13 +2993,13 @@ if (total_args != 2) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } PyInterpreterState *interp = tstate->interp; if (callable_o != interp->callable_cache.isinstance) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); _PyStackRef cls_stackref = arguments[1]; @@ -2282,7 +3008,7 @@ int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); stack_pointer = _PyFrame_GetStackPointer(frame); if (retval < 0) { - goto error; + JUMP_TO_LABEL(error); } res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -2298,6 +3024,10 @@ } TARGET(CALL_KW) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW); @@ -2393,7 +3123,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - goto error; + JUMP_TO_LABEL(error); } assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4 ; @@ -2410,7 +3140,7 @@ PyStackRef_CLOSE(kwnames); stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } stack_pointer[-1] = kwnames; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2450,7 +3180,7 @@ if (res_o == NULL) { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2461,8 +3191,13 @@ } TARGET(CALL_KW_BOUND_METHOD) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); @@ -2478,7 +3213,7 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - goto PREDICTED_CALL_KW; + JUMP_TO_PREDICTED(CALL_KW); } } // _CHECK_METHOD_VERSION_KW @@ -2490,23 +3225,23 @@ if (Py_TYPE(callable_o) != &PyMethod_Type) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - goto PREDICTED_CALL_KW; + JUMP_TO_PREDICTED(CALL_KW); } PyObject *func = ((PyMethodObject *)callable_o)->im_func; if (!PyFunction_Check(func)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - goto PREDICTED_CALL_KW; + JUMP_TO_PREDICTED(CALL_KW); } if (((PyFunctionObject *)func)->func_version != func_version) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - goto PREDICTED_CALL_KW; + JUMP_TO_PREDICTED(CALL_KW); } if (!PyStackRef_IsNull(null[0])) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - goto PREDICTED_CALL_KW; + JUMP_TO_PREDICTED(CALL_KW); } } // _EXPAND_METHOD_KW @@ -2557,7 +3292,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - goto error; + JUMP_TO_LABEL(error); } new_frame = temp; } @@ -2589,6 +3324,14 @@ } TARGET(CALL_KW_NON_PY) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_NON_PY); @@ -2608,12 +3351,12 @@ if (PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - goto PREDICTED_CALL_KW; + JUMP_TO_PREDICTED(CALL_KW); } if (Py_TYPE(callable_o) == &PyMethod_Type) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - goto PREDICTED_CALL_KW; + JUMP_TO_PREDICTED(CALL_KW); } } // _CALL_KW_NON_PY @@ -2642,7 +3385,7 @@ PyStackRef_CLOSE(kwnames); stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); @@ -2667,7 +3410,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2683,7 +3426,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2696,8 +3439,13 @@ } TARGET(CALL_KW_PY) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_PY); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); @@ -2712,7 +3460,7 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - goto PREDICTED_CALL_KW; + JUMP_TO_PREDICTED(CALL_KW); } } // _CHECK_FUNCTION_VERSION_KW @@ -2723,13 +3471,13 @@ if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - goto PREDICTED_CALL_KW; + JUMP_TO_PREDICTED(CALL_KW); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - goto PREDICTED_CALL_KW; + JUMP_TO_PREDICTED(CALL_KW); } } // _PY_FRAME_KW @@ -2766,7 +3514,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - goto error; + JUMP_TO_LABEL(error); } new_frame = temp; } @@ -2798,6 +3546,14 @@ } TARGET(CALL_LEN) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LEN); @@ -2821,13 +3577,13 @@ if (total_args != 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } PyInterpreterState *interp = tstate->interp; if (callable_o != interp->callable_cache.len) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); _PyStackRef arg_stackref = args[0]; @@ -2836,7 +3592,7 @@ Py_ssize_t len_i = PyObject_Length(arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (len_i < 0) { - goto error; + JUMP_TO_LABEL(error); } PyObject *res_o = PyLong_FromSsize_t(len_i); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -2859,6 +3615,14 @@ } TARGET(CALL_LIST_APPEND) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LIST_APPEND); @@ -2878,18 +3642,18 @@ if (callable_o != interp->callable_cache.list_append) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } assert(self_o != NULL); if (!PyList_Check(self_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } if (!LOCK_OBJECT(self_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); @@ -2905,7 +3669,7 @@ PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } #if TIER_ONE // Skip the following POP_TOP. This is done here in tier one, and @@ -2917,6 +3681,14 @@ } TARGET(CALL_METHOD_DESCRIPTOR_FAST) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); @@ -2944,19 +3716,19 @@ if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); int nargs = total_args - 1; @@ -2969,7 +3741,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFast cfunc = @@ -2986,7 +3758,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3002,7 +3774,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -3015,6 +3787,14 @@ } TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); @@ -3041,20 +3821,20 @@ if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } PyTypeObject *d_type = method->d_common.d_type; PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); if (!Py_IS_TYPE(self, d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); int nargs = total_args - 1; @@ -3067,7 +3847,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFastWithKeywords cfunc = @@ -3084,7 +3864,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3100,7 +3880,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -3113,6 +3893,14 @@ } TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); @@ -3138,13 +3926,13 @@ if (total_args != 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } PyMethodDef *meth = method->d_method; _PyStackRef self_stackref = args[0]; @@ -3152,18 +3940,18 @@ if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } if (meth->ml_flags != METH_NOARGS) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } // CPython promises to check all non-vectorcall function calls. if (tstate->c_recursion_remaining <= 0) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; @@ -3182,7 +3970,7 @@ PyStackRef_CLOSE(callable[0]); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3198,7 +3986,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3211,6 +3999,14 @@ } TARGET(CALL_METHOD_DESCRIPTOR_O) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); @@ -3237,24 +4033,24 @@ if (total_args != 2) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } // CPython promises to check all non-vectorcall function calls. if (tstate->c_recursion_remaining <= 0) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } _PyStackRef arg_stackref = arguments[1]; _PyStackRef self_stackref = arguments[0]; @@ -3262,7 +4058,7 @@ method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; @@ -3282,7 +4078,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3298,7 +4094,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -3311,6 +4107,14 @@ } TARGET(CALL_NON_PY_GENERAL) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_NON_PY_GENERAL); @@ -3329,12 +4133,12 @@ if (PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } if (Py_TYPE(callable_o) == &PyMethod_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } } // _CALL_NON_PY_GENERAL @@ -3361,7 +4165,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( @@ -3379,7 +4183,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3395,7 +4199,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -3408,8 +4212,13 @@ } TARGET(CALL_PY_EXACT_ARGS) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3423,7 +4232,7 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } } // _CHECK_FUNCTION_VERSION @@ -3434,13 +4243,13 @@ if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } } // _CHECK_FUNCTION_EXACT_ARGS @@ -3453,7 +4262,7 @@ if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } } // _CHECK_STACK_SPACE @@ -3464,12 +4273,12 @@ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } } // _INIT_CALL_PY_EXACT_ARGS @@ -3514,8 +4323,13 @@ } TARGET(CALL_PY_GENERAL) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3529,7 +4343,7 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } } // _CHECK_FUNCTION_VERSION @@ -3540,13 +4354,13 @@ if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } } // _PY_FRAME_GENERAL @@ -3573,7 +4387,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - goto error; + JUMP_TO_LABEL(error); } new_frame = temp; } @@ -3605,6 +4419,14 @@ } TARGET(CALL_STR_1) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_STR_1); @@ -3626,12 +4448,12 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } if (callable_o != (PyObject *)&PyUnicode_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3643,7 +4465,7 @@ PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3659,7 +4481,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3672,6 +4494,14 @@ } TARGET(CALL_TUPLE_1) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TUPLE_1); @@ -3693,12 +4523,12 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } if (callable_o != (PyObject *)&PyTuple_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3710,7 +4540,7 @@ PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3726,7 +4556,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3739,6 +4569,14 @@ } TARGET(CALL_TYPE_1) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TYPE_1); @@ -3758,12 +4596,12 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } if (callable_o != (PyObject *)&PyType_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - goto PREDICTED_CALL; + JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); @@ -3777,6 +4615,10 @@ } TARGET(CHECK_EG_MATCH) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EG_MATCH); @@ -3794,7 +4636,7 @@ if (err < 0) { PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } PyObject *match_o = NULL; PyObject *rest_o = NULL; @@ -3805,11 +4647,11 @@ PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); if (res < 0) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } assert((match_o == NULL) == (rest_o == NULL)); if (match_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } if (!Py_IsNone(match_o)) { stack_pointer += -2; @@ -3828,6 +4670,10 @@ } TARGET(CHECK_EXC_MATCH) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EXC_MATCH); @@ -3844,7 +4690,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { PyStackRef_CLOSE(right); - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyErr_GivenExceptionMatches(left_o, right_o); @@ -3856,8 +4702,13 @@ } TARGET(CLEANUP_THROW) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CLEANUP_THROW); _PyStackRef sub_iter_st; @@ -3888,7 +4739,7 @@ _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } stack_pointer[-3] = none; stack_pointer[-2] = value; @@ -3898,6 +4749,10 @@ } TARGET(COMPARE_OP) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP); @@ -3936,7 +4791,7 @@ PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } if (oparg & 16) { stack_pointer += -2; @@ -3946,7 +4801,7 @@ Py_DECREF(res_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_bool < 0) { - goto error; + JUMP_TO_LABEL(error); } res = res_bool ? PyStackRef_True : PyStackRef_False; } @@ -3963,6 +4818,14 @@ } TARGET(COMPARE_OP_FLOAT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_FLOAT); @@ -3979,12 +4842,12 @@ if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - goto PREDICTED_COMPARE_OP; + JUMP_TO_PREDICTED(COMPARE_OP); } if (!PyFloat_CheckExact(right_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - goto PREDICTED_COMPARE_OP; + JUMP_TO_PREDICTED(COMPARE_OP); } } /* Skip 1 cache entry */ @@ -4009,6 +4872,14 @@ } TARGET(COMPARE_OP_INT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_INT); @@ -4025,12 +4896,12 @@ if (!PyLong_CheckExact(left_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - goto PREDICTED_COMPARE_OP; + JUMP_TO_PREDICTED(COMPARE_OP); } if (!PyLong_CheckExact(right_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - goto PREDICTED_COMPARE_OP; + JUMP_TO_PREDICTED(COMPARE_OP); } } /* Skip 1 cache entry */ @@ -4041,12 +4912,12 @@ if (!_PyLong_IsCompact((PyLongObject *)left_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - goto PREDICTED_COMPARE_OP; + JUMP_TO_PREDICTED(COMPARE_OP); } if (!_PyLong_IsCompact((PyLongObject *)right_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - goto PREDICTED_COMPARE_OP; + JUMP_TO_PREDICTED(COMPARE_OP); } STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && @@ -4067,6 +4938,14 @@ } TARGET(COMPARE_OP_STR) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_STR); @@ -4083,12 +4962,12 @@ if (!PyUnicode_CheckExact(left_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - goto PREDICTED_COMPARE_OP; + JUMP_TO_PREDICTED(COMPARE_OP); } if (!PyUnicode_CheckExact(right_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - goto PREDICTED_COMPARE_OP; + JUMP_TO_PREDICTED(COMPARE_OP); } } /* Skip 1 cache entry */ @@ -4114,6 +4993,10 @@ } TARGET(CONTAINS_OP) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP); @@ -4151,7 +5034,7 @@ PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res < 0) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } @@ -4162,6 +5045,14 @@ } TARGET(CONTAINS_OP_DICT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_DICT); @@ -4177,7 +5068,7 @@ if (!PyDict_CheckExact(right_o)) { UPDATE_MISS_STATS(CONTAINS_OP); assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); - goto PREDICTED_CONTAINS_OP; + JUMP_TO_PREDICTED(CONTAINS_OP); } STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4186,7 +5077,7 @@ PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res < 0) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; @@ -4196,6 +5087,14 @@ } TARGET(CONTAINS_OP_SET) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_SET); @@ -4211,7 +5110,7 @@ if (!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o))) { UPDATE_MISS_STATS(CONTAINS_OP); assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); - goto PREDICTED_CONTAINS_OP; + JUMP_TO_PREDICTED(CONTAINS_OP); } STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! @@ -4221,7 +5120,7 @@ PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res < 0) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; @@ -4231,6 +5130,10 @@ } TARGET(CONVERT_VALUE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CONVERT_VALUE); @@ -4249,7 +5152,7 @@ PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); if (result_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } result = PyStackRef_FromPyObjectSteal(result_o); stack_pointer[0] = result; @@ -4259,6 +5162,10 @@ } TARGET(COPY) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY); @@ -4274,6 +5181,10 @@ } TARGET(COPY_FREE_VARS) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY_FREE_VARS); @@ -4292,6 +5203,10 @@ } TARGET(DELETE_ATTR) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_ATTR); @@ -4303,7 +5218,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); if (err) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -4311,6 +5226,10 @@ } TARGET(DELETE_DEREF) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_DEREF); @@ -4322,7 +5241,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(oldobj); @@ -4331,6 +5250,10 @@ } TARGET(DELETE_FAST) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_FAST); @@ -4342,13 +5265,17 @@ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } SETLOCAL(oparg, PyStackRef_NULL); DISPATCH(); } TARGET(DELETE_GLOBAL) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_GLOBAL); @@ -4358,19 +5285,23 @@ stack_pointer = _PyFrame_GetStackPointer(frame); // Can't use ERROR_IF here. if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } if (err == 0) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } DISPATCH(); } TARGET(DELETE_NAME) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_NAME); @@ -4382,7 +5313,7 @@ _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_DelItem(ns, name); @@ -4394,12 +5325,16 @@ NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } DISPATCH(); } TARGET(DELETE_SUBSCR) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_SUBSCR); @@ -4415,7 +5350,7 @@ PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); if (err) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -4423,6 +5358,10 @@ } TARGET(DICT_MERGE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_MERGE); @@ -4443,7 +5382,7 @@ _PyEval_FormatKwargsError(tstate, callable_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(update); - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } PyStackRef_CLOSE(update); stack_pointer += -1; @@ -4452,6 +5391,10 @@ } TARGET(DICT_UPDATE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_UPDATE); @@ -4476,7 +5419,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(update); - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } PyStackRef_CLOSE(update); stack_pointer += -1; @@ -4485,8 +5428,13 @@ } TARGET(END_ASYNC_FOR) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_ASYNC_FOR); _PyStackRef awaitable_st; @@ -4508,7 +5456,7 @@ _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -4516,6 +5464,10 @@ } TARGET(END_FOR) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ next_instr += 1; INSTRUCTION_STATS(END_FOR); _PyStackRef value; @@ -4534,6 +5486,10 @@ } TARGET(END_SEND) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_SEND); @@ -4552,8 +5508,13 @@ } TARGET(ENTER_EXECUTOR) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(ENTER_EXECUTOR); opcode = ENTER_EXECUTOR; @@ -4586,6 +5547,10 @@ } TARGET(EXIT_INIT_CHECK) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXIT_INIT_CHECK); @@ -4598,7 +5563,7 @@ "__init__() should return None, not '%.200s'", Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -4606,6 +5571,10 @@ } TARGET(EXTENDED_ARG) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXTENDED_ARG); @@ -4618,6 +5587,10 @@ } TARGET(FORMAT_SIMPLE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_SIMPLE); @@ -4637,7 +5610,7 @@ PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4653,6 +5626,10 @@ } TARGET(FORMAT_WITH_SPEC) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_WITH_SPEC); @@ -4667,7 +5644,7 @@ PyStackRef_CLOSE(value); PyStackRef_CLOSE(fmt_spec); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -4677,6 +5654,10 @@ } TARGET(FOR_ITER) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER); @@ -4715,7 +5696,7 @@ int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); stack_pointer = _PyFrame_GetStackPointer(frame); if (!matches) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_MonitorRaise(tstate, frame, this_instr); @@ -4739,6 +5720,14 @@ } TARGET(FOR_ITER_GEN) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_GEN); @@ -4752,7 +5741,7 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - goto PREDICTED_FOR_ITER; + JUMP_TO_PREDICTED(FOR_ITER); } } // _FOR_ITER_GEN_FRAME @@ -4762,12 +5751,12 @@ if (Py_TYPE(gen) != &PyGen_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - goto PREDICTED_FOR_ITER; + JUMP_TO_PREDICTED(FOR_ITER); } if (gen->gi_frame_state >= FRAME_EXECUTING) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - goto PREDICTED_FOR_ITER; + JUMP_TO_PREDICTED(FOR_ITER); } STAT_INC(FOR_ITER, hit); gen_frame = &gen->gi_iframe; @@ -4799,6 +5788,14 @@ } TARGET(FOR_ITER_LIST) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_LIST); @@ -4812,7 +5809,7 @@ if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - goto PREDICTED_FOR_ITER; + JUMP_TO_PREDICTED(FOR_ITER); } } // _ITER_JUMP_LIST @@ -4854,6 +5851,14 @@ } TARGET(FOR_ITER_RANGE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_RANGE); @@ -4868,7 +5873,7 @@ if (Py_TYPE(r) != &PyRangeIter_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - goto PREDICTED_FOR_ITER; + JUMP_TO_PREDICTED(FOR_ITER); } } // _ITER_JUMP_RANGE @@ -4892,7 +5897,7 @@ r->len--; PyObject *res = PyLong_FromLong(value); if (res == NULL) { - goto error; + JUMP_TO_LABEL(error); } next = PyStackRef_FromPyObjectSteal(res); } @@ -4903,6 +5908,14 @@ } TARGET(FOR_ITER_TUPLE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_TUPLE); @@ -4916,7 +5929,7 @@ if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - goto PREDICTED_FOR_ITER; + JUMP_TO_PREDICTED(FOR_ITER); } } // _ITER_JUMP_TUPLE @@ -4955,6 +5968,10 @@ } TARGET(GET_AITER) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AITER); @@ -4976,14 +5993,14 @@ type->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } _PyFrame_SetStackPointer(frame, stack_pointer); iter_o = (*getter)(obj_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); if (iter_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { @@ -4996,7 +6013,7 @@ Py_TYPE(iter_o)->tp_name); Py_DECREF(iter_o); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; @@ -5004,6 +6021,10 @@ } TARGET(GET_ANEXT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ANEXT); @@ -5014,7 +6035,7 @@ PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); stack_pointer = _PyFrame_GetStackPointer(frame); if (awaitable_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); stack_pointer[0] = awaitable; @@ -5024,6 +6045,10 @@ } TARGET(GET_AWAITABLE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AWAITABLE); @@ -5035,7 +6060,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); if (iter_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; @@ -5043,6 +6068,10 @@ } TARGET(GET_ITER) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ITER); @@ -5055,7 +6084,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); if (iter_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; @@ -5063,6 +6092,10 @@ } TARGET(GET_LEN) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_LEN); @@ -5074,11 +6107,11 @@ Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); stack_pointer = _PyFrame_GetStackPointer(frame); if (len_i < 0) { - goto error; + JUMP_TO_LABEL(error); } PyObject *len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } len = PyStackRef_FromPyObjectSteal(len_o); stack_pointer[0] = len; @@ -5088,6 +6121,10 @@ } TARGET(GET_YIELD_FROM_ITER) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_YIELD_FROM_ITER); @@ -5106,7 +6143,7 @@ "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } iter = iterable; } @@ -5120,7 +6157,7 @@ PyObject *iter_o = PyObject_GetIter(iterable_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); PyStackRef_CLOSE(iterable); @@ -5131,6 +6168,10 @@ } TARGET(IMPORT_FROM) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_FROM); @@ -5142,7 +6183,7 @@ PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; @@ -5152,6 +6193,10 @@ } TARGET(IMPORT_NAME) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_NAME); @@ -5169,7 +6214,7 @@ PyStackRef_CLOSE(level); PyStackRef_CLOSE(fromlist); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -5179,8 +6224,13 @@ } TARGET(INSTRUMENTED_CALL) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL); opcode = INSTRUMENTED_CALL; @@ -5236,7 +6286,7 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } } // _DO_CALL @@ -5270,7 +6320,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - goto error; + JUMP_TO_LABEL(error); } frame->return_offset = 4 ; DISPATCH_INLINED(new_frame); @@ -5285,7 +6335,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( @@ -5324,7 +6374,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -5340,7 +6390,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -5353,8 +6403,13 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); opcode = INSTRUMENTED_CALL_FUNCTION_EX; @@ -5383,13 +6438,13 @@ int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *tuple_o = PySequence_Tuple(callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (tuple_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } kwargs_out = kwargs_in; stack_pointer += -2; @@ -5430,7 +6485,7 @@ frame, this_instr, func, arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); result_o = PyObject_Call(func, callargs, kwargs); @@ -5477,7 +6532,7 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { - goto error; + JUMP_TO_LABEL(error); } assert( 1 == 1); frame->return_offset = 1; @@ -5509,7 +6564,7 @@ PyStackRef_CLOSE(func_st); stack_pointer = _PyFrame_GetStackPointer(frame); if (result_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } result = PyStackRef_FromPyObjectSteal(result_o); } @@ -5525,7 +6580,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -5538,8 +6593,13 @@ } TARGET(INSTRUMENTED_CALL_KW) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); opcode = INSTRUMENTED_CALL_KW; @@ -5579,7 +6639,7 @@ frame, this_instr, function, arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } } // _MAYBE_EXPAND_METHOD_KW @@ -5641,7 +6701,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - goto error; + JUMP_TO_LABEL(error); } assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4 ; @@ -5658,7 +6718,7 @@ PyStackRef_CLOSE(kwnames); stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } stack_pointer[-1] = kwnames; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5698,7 +6758,7 @@ if (res_o == NULL) { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -5709,6 +6769,10 @@ } TARGET(INSTRUMENTED_END_FOR) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; next_instr += 1; @@ -5724,7 +6788,7 @@ int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } } PyStackRef_CLOSE(value); @@ -5734,8 +6798,13 @@ } TARGET(INSTRUMENTED_END_SEND) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_SEND); _PyStackRef receiver; @@ -5749,7 +6818,7 @@ int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } } val = value; @@ -5763,8 +6832,13 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); /* Skip 1 cache entry */ @@ -5783,7 +6857,7 @@ int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); stack_pointer = _PyFrame_GetStackPointer(frame); if (!matches) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_MonitorRaise(tstate, frame, this_instr); @@ -5800,8 +6874,13 @@ } TARGET(INSTRUMENTED_INSTRUCTION) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); opcode = INSTRUMENTED_INSTRUCTION; @@ -5810,7 +6889,7 @@ tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); if (next_opcode < 0) { - goto error; + JUMP_TO_LABEL(error); } next_instr = this_instr; if (_PyOpcode_Caches[next_opcode]) { @@ -5822,8 +6901,13 @@ } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); /* Skip 1 cache entry */ @@ -5836,7 +6920,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } } } @@ -5848,8 +6932,13 @@ } TARGET(INSTRUMENTED_JUMP_FORWARD) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); @@ -5857,9 +6946,14 @@ } TARGET(INSTRUMENTED_LINE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_LINE); opcode = INSTRUMENTED_LINE; @@ -5878,7 +6972,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (original_opcode < 0) { next_instr = this_instr+1; - goto error; + JUMP_TO_LABEL(error); } next_instr = frame->instr_ptr; if (next_instr != this_instr) { @@ -5896,8 +6990,13 @@ } TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); opcode = INSTRUMENTED_LOAD_SUPER_ATTR; @@ -5926,7 +7025,7 @@ PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); - goto pop_3_error; + JUMP_TO_LABEL(pop_3_error); } } // we make no attempt to optimize here; specializations should @@ -5959,7 +7058,7 @@ PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); if (super == NULL) { - goto pop_3_error; + JUMP_TO_LABEL(pop_3_error); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); stack_pointer += -3; @@ -5969,7 +7068,7 @@ Py_DECREF(super); stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } attr = PyStackRef_FromPyObjectSteal(attr_o); } @@ -5985,9 +7084,14 @@ } TARGET(INSTRUMENTED_NOT_TAKEN) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); (void)this_instr; // INSTRUMENTED_JUMP requires this_instr @@ -5996,9 +7100,14 @@ } TARGET(INSTRUMENTED_POP_ITER) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); _PyStackRef iter; @@ -6013,8 +7122,13 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); /* Skip 1 cache entry */ @@ -6029,8 +7143,13 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); /* Skip 1 cache entry */ @@ -6049,8 +7168,13 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); /* Skip 1 cache entry */ @@ -6067,8 +7191,13 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); /* Skip 1 cache entry */ @@ -6083,8 +7212,13 @@ } TARGET(INSTRUMENTED_RESUME) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RESUME); // _LOAD_BYTECODE @@ -6097,7 +7231,7 @@ _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); if (bytecode == NULL) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); @@ -6121,7 +7255,7 @@ int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } next_instr = this_instr; DISPATCH(); @@ -6138,7 +7272,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } } } @@ -6150,7 +7284,7 @@ tstate, oparg > 0, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } if (frame->instr_ptr != this_instr) { /* Instrumentation has jumped */ @@ -6161,8 +7295,13 @@ } TARGET(INSTRUMENTED_RETURN_VALUE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); _PyStackRef val; @@ -6177,7 +7316,7 @@ frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } } // _RETURN_VALUE @@ -6206,8 +7345,13 @@ } TARGET(INSTRUMENTED_YIELD_VALUE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); _PyStackRef val; @@ -6222,7 +7366,7 @@ frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } if (frame->instr_ptr != this_instr) { next_instr = frame->instr_ptr; @@ -6273,6 +7417,10 @@ } TARGET(INTERPRETER_EXIT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INTERPRETER_EXIT); @@ -6292,6 +7440,10 @@ } TARGET(IS_OP) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IS_OP); @@ -6311,6 +7463,10 @@ } TARGET(JUMP_BACKWARD) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD); @@ -6338,7 +7494,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } } } @@ -6356,8 +7512,13 @@ } TARGET(JUMP_BACKWARD_JIT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD_JIT); static_assert(1 == 1, "incorrect cache size"); @@ -6371,7 +7532,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } } } @@ -6403,7 +7564,7 @@ if (optimized <= 0) { this_instr[1].counter = restart_backoff_counter(counter); if (optimized < 0) { - goto error; + JUMP_TO_LABEL(error); } } else { @@ -6424,6 +7585,10 @@ } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); @@ -6438,6 +7603,10 @@ } TARGET(JUMP_BACKWARD_NO_JIT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); @@ -6452,7 +7621,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } } } @@ -6470,6 +7639,10 @@ } TARGET(JUMP_FORWARD) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_FORWARD); @@ -6478,6 +7651,10 @@ } TARGET(LIST_APPEND) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_APPEND); @@ -6488,7 +7665,7 @@ int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), PyStackRef_AsPyObjectSteal(v)); if (err < 0) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -6496,6 +7673,10 @@ } TARGET(LIST_EXTEND) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_EXTEND); @@ -6523,7 +7704,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(iterable_st); - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } assert(Py_IsNone(none_val)); PyStackRef_CLOSE(iterable_st); @@ -6533,6 +7714,10 @@ } TARGET(LOAD_ATTR) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR); @@ -6589,7 +7774,7 @@ */ PyStackRef_CLOSE(owner); if (attr_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } self_or_null[0] = PyStackRef_NULL; } @@ -6601,7 +7786,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); if (attr_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } } attr = PyStackRef_FromPyObjectSteal(attr_o); @@ -6613,8 +7798,13 @@ } TARGET(LOAD_ATTR_CLASS) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6630,13 +7820,13 @@ if (!PyType_Check(owner_o)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } /* Skip 2 cache entries */ @@ -6660,8 +7850,13 @@ } TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6677,13 +7872,13 @@ if (!PyType_Check(owner_o)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } // _GUARD_TYPE_VERSION @@ -6694,7 +7889,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } // _LOAD_ATTR_CLASS @@ -6717,8 +7912,13 @@ } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6733,14 +7933,14 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; @@ -6748,14 +7948,14 @@ if (f->func_version != func_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 2); if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); @@ -6770,8 +7970,13 @@ } TARGET(LOAD_ATTR_INSTANCE_VALUE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6788,7 +7993,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } // _CHECK_MANAGED_OBJECT_HAS_VALUES @@ -6799,7 +8004,7 @@ if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } // _LOAD_ATTR_INSTANCE_VALUE @@ -6811,14 +8016,14 @@ if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } #ifdef Py_GIL_DISABLED if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } #else @@ -6842,8 +8047,13 @@ } TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6860,7 +8070,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } // _CHECK_ATTR_METHOD_LAZY_DICT @@ -6872,7 +8082,7 @@ if (dict != NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } /* Skip 1 cache entry */ @@ -6894,8 +8104,13 @@ } TARGET(LOAD_ATTR_METHOD_NO_DICT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6912,7 +8127,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } /* Skip 2 cache entries */ @@ -6935,8 +8150,13 @@ } TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6953,7 +8173,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT @@ -6964,7 +8184,7 @@ if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } // _GUARD_KEYS_VERSION @@ -6976,7 +8196,7 @@ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } // _LOAD_ATTR_METHOD_WITH_VALUES @@ -6998,8 +8218,13 @@ } TARGET(LOAD_ATTR_MODULE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -7016,7 +8241,7 @@ if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); @@ -7024,7 +8249,7 @@ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } mod_keys = keys; } @@ -7039,7 +8264,7 @@ if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); @@ -7047,7 +8272,7 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } #else @@ -7072,8 +8297,13 @@ } TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -7089,7 +8319,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } /* Skip 2 cache entries */ @@ -7108,8 +8338,13 @@ } TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -7125,7 +8360,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT @@ -7136,7 +8371,7 @@ if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } // _GUARD_KEYS_VERSION @@ -7148,7 +8383,7 @@ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES @@ -7165,8 +8400,13 @@ } TARGET(LOAD_ATTR_PROPERTY) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -7178,7 +8418,7 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } // _GUARD_TYPE_VERSION @@ -7190,7 +8430,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } /* Skip 2 cache entries */ @@ -7204,22 +8444,22 @@ if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } if (code->co_kwonlyargcount) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } if (code->co_argcount != 1) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } STAT_INC(LOAD_ATTR, hit); new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); @@ -7255,8 +8495,13 @@ } TARGET(LOAD_ATTR_SLOT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -7273,7 +8518,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } // _LOAD_ATTR_SLOT @@ -7285,14 +8530,14 @@ if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); if (!increfed) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } #else attr = PyStackRef_FromPyObjectNew(attr_o); @@ -7313,8 +8558,13 @@ } TARGET(LOAD_ATTR_WITH_HINT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -7332,7 +8582,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } // _CHECK_ATTR_WITH_HINT @@ -7343,7 +8593,7 @@ if (dict_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } assert(PyDict_CheckExact((PyObject *)dict_o)); dict = dict_o; @@ -7356,7 +8606,7 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } if (hint >= (size_t)dict->ma_keys->dk_nentries) { @@ -7364,7 +8614,7 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); @@ -7373,7 +8623,7 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; @@ -7382,7 +8632,7 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } attr_o = ep->me_value; @@ -7391,7 +8641,7 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - goto PREDICTED_LOAD_ATTR; + JUMP_TO_PREDICTED(LOAD_ATTR); } } STAT_INC(LOAD_ATTR, hit); @@ -7412,6 +8662,10 @@ } TARGET(LOAD_BUILD_CLASS) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_BUILD_CLASS); @@ -7421,14 +8675,14 @@ int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } if (bc_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } bc = PyStackRef_FromPyObjectSteal(bc_o); stack_pointer[0] = bc; @@ -7438,6 +8692,10 @@ } TARGET(LOAD_COMMON_CONSTANT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); @@ -7460,6 +8718,10 @@ } TARGET(LOAD_CONST) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); @@ -7493,6 +8755,10 @@ } TARGET(LOAD_CONST_IMMORTAL) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST_IMMORTAL); @@ -7508,6 +8774,10 @@ } TARGET(LOAD_CONST_MORTAL) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST_MORTAL); @@ -7522,6 +8792,10 @@ } TARGET(LOAD_DEREF) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_DEREF); @@ -7532,7 +8806,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; @@ -7542,6 +8816,10 @@ } TARGET(LOAD_FAST) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST); @@ -7555,6 +8833,10 @@ } TARGET(LOAD_FAST_AND_CLEAR) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); @@ -7569,6 +8851,10 @@ } TARGET(LOAD_FAST_CHECK) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_CHECK); @@ -7581,7 +8867,7 @@ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } value = PyStackRef_DUP(value_s); stack_pointer[0] = value; @@ -7591,6 +8877,10 @@ } TARGET(LOAD_FAST_LOAD_FAST) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); @@ -7608,6 +8898,10 @@ } TARGET(LOAD_FROM_DICT_OR_DEREF) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); @@ -7624,7 +8918,7 @@ int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } if (!value_o) { PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); @@ -7633,7 +8927,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } } stack_pointer += -1; @@ -7649,6 +8943,10 @@ } TARGET(LOAD_FROM_DICT_OR_GLOBALS) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); @@ -7662,7 +8960,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(mod_or_class_dict); if (err < 0) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } if (v_o == NULL) { if (PyDict_CheckExact(GLOBALS()) @@ -7684,7 +8982,7 @@ NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); } - goto error; + JUMP_TO_LABEL(error); } } else { @@ -7696,7 +8994,7 @@ int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } if (v_o == NULL) { /* namespace 2: builtins */ @@ -7704,7 +9002,7 @@ int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } if (v_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -7712,7 +9010,7 @@ tstate, PyExc_NameError, NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } } } @@ -7725,6 +9023,10 @@ } TARGET(LOAD_GLOBAL) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL); @@ -7761,7 +9063,7 @@ _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*res)) { - goto error; + JUMP_TO_LABEL(error); } } // _PUSH_NULL_CONDITIONAL @@ -7775,8 +9077,13 @@ } TARGET(LOAD_GLOBAL_BUILTIN) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); @@ -7791,13 +9098,13 @@ if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - goto PREDICTED_LOAD_GLOBAL; + JUMP_TO_PREDICTED(LOAD_GLOBAL); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - goto PREDICTED_LOAD_GLOBAL; + JUMP_TO_PREDICTED(LOAD_GLOBAL); } assert(DK_IS_UNICODE(keys)); } @@ -7808,13 +9115,13 @@ if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - goto PREDICTED_LOAD_GLOBAL; + JUMP_TO_PREDICTED(LOAD_GLOBAL); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - goto PREDICTED_LOAD_GLOBAL; + JUMP_TO_PREDICTED(LOAD_GLOBAL); } builtins_keys = keys; assert(DK_IS_UNICODE(builtins_keys)); @@ -7827,14 +9134,14 @@ if (res_o == NULL) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - goto PREDICTED_LOAD_GLOBAL; + JUMP_TO_PREDICTED(LOAD_GLOBAL); } #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); if (!increfed) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - goto PREDICTED_LOAD_GLOBAL; + JUMP_TO_PREDICTED(LOAD_GLOBAL); } #else Py_INCREF(res_o); @@ -7854,8 +9161,13 @@ } TARGET(LOAD_GLOBAL_MODULE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); @@ -7870,13 +9182,13 @@ if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - goto PREDICTED_LOAD_GLOBAL; + JUMP_TO_PREDICTED(LOAD_GLOBAL); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - goto PREDICTED_LOAD_GLOBAL; + JUMP_TO_PREDICTED(LOAD_GLOBAL); } globals_keys = keys; assert(DK_IS_UNICODE(globals_keys)); @@ -7890,14 +9202,14 @@ if (res_o == NULL) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - goto PREDICTED_LOAD_GLOBAL; + JUMP_TO_PREDICTED(LOAD_GLOBAL); } #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); if (!increfed) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - goto PREDICTED_LOAD_GLOBAL; + JUMP_TO_PREDICTED(LOAD_GLOBAL); } #else Py_INCREF(res_o); @@ -7917,6 +9229,10 @@ } TARGET(LOAD_LOCALS) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_LOCALS); @@ -7927,7 +9243,7 @@ _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } locals = PyStackRef_FromPyObjectNew(l); stack_pointer[0] = locals; @@ -7937,6 +9253,10 @@ } TARGET(LOAD_NAME) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_NAME); @@ -7946,7 +9266,7 @@ PyObject *v_o = _PyEval_LoadName(tstate, frame, name); stack_pointer = _PyFrame_GetStackPointer(frame); if (v_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; @@ -7956,6 +9276,10 @@ } TARGET(LOAD_SMALL_INT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SMALL_INT); @@ -7970,6 +9294,10 @@ } TARGET(LOAD_SPECIAL) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SPECIAL); @@ -7994,7 +9322,7 @@ Py_TYPE(owner_o)->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); } - goto error; + JUMP_TO_LABEL(error); } attr = PyStackRef_FromPyObjectSteal(attr_o); self_or_null = self_or_null_o == NULL ? @@ -8007,6 +9335,10 @@ } TARGET(LOAD_SUPER_ATTR) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR); @@ -8055,7 +9387,7 @@ PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); - goto pop_3_error; + JUMP_TO_LABEL(pop_3_error); } } // we make no attempt to optimize here; specializations should @@ -8088,7 +9420,7 @@ PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); if (super == NULL) { - goto pop_3_error; + JUMP_TO_LABEL(pop_3_error); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); stack_pointer += -3; @@ -8098,7 +9430,7 @@ Py_DECREF(super); stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } attr = PyStackRef_FromPyObjectSteal(attr_o); } @@ -8114,6 +9446,14 @@ } TARGET(LOAD_SUPER_ATTR_ATTR) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); @@ -8133,12 +9473,12 @@ if (global_super != (PyObject *)&PySuper_Type) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - goto PREDICTED_LOAD_SUPER_ATTR; + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); } if (!PyType_Check(class)) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - goto PREDICTED_LOAD_SUPER_ATTR; + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); @@ -8149,7 +9489,7 @@ PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); if (attr == NULL) { - goto pop_3_error; + JUMP_TO_LABEL(pop_3_error); } attr_st = PyStackRef_FromPyObjectSteal(attr); stack_pointer[-3] = attr_st; @@ -8159,6 +9499,14 @@ } TARGET(LOAD_SUPER_ATTR_METHOD) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); @@ -8179,12 +9527,12 @@ if (global_super != (PyObject *)&PySuper_Type) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - goto PREDICTED_LOAD_SUPER_ATTR; + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); } if (!PyType_Check(class)) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - goto PREDICTED_LOAD_SUPER_ATTR; + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); @@ -8195,7 +9543,7 @@ Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } if (method_found) { self_or_null = self_st; // transfer ownership @@ -8220,6 +9568,10 @@ } TARGET(MAKE_CELL) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_CELL); @@ -8228,13 +9580,17 @@ PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *cell = PyCell_New(initial); if (cell == NULL) { - goto error; + JUMP_TO_LABEL(error); } SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell)); DISPATCH(); } TARGET(MAKE_FUNCTION) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_FUNCTION); @@ -8252,7 +9608,7 @@ PyStackRef_CLOSE(codeobj_st); stack_pointer = _PyFrame_GetStackPointer(frame); if (func_obj == NULL) { - goto error; + JUMP_TO_LABEL(error); } _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); @@ -8264,6 +9620,10 @@ } TARGET(MAP_ADD) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAP_ADD); @@ -8285,7 +9645,7 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -8293,6 +9653,10 @@ } TARGET(MATCH_CLASS) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_CLASS); @@ -8321,7 +9685,7 @@ } else { if (_PyErr_Occurred(tstate)) { - goto pop_3_error; + JUMP_TO_LABEL(pop_3_error); } // Error! attrs = PyStackRef_None; // Failure! @@ -8333,6 +9697,10 @@ } TARGET(MATCH_KEYS) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_KEYS); @@ -8347,7 +9715,7 @@ PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); stack_pointer = _PyFrame_GetStackPointer(frame); if (values_or_none_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); stack_pointer[0] = values_or_none; @@ -8357,6 +9725,10 @@ } TARGET(MATCH_MAPPING) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_MAPPING); @@ -8372,6 +9744,10 @@ } TARGET(MATCH_SEQUENCE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_SEQUENCE); @@ -8387,6 +9763,10 @@ } TARGET(NOP) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOP); @@ -8394,6 +9774,10 @@ } TARGET(NOT_TAKEN) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOT_TAKEN); @@ -8401,6 +9785,10 @@ } TARGET(POP_EXCEPT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_EXCEPT); @@ -8418,6 +9806,10 @@ } TARGET(POP_ITER) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_ITER); @@ -8430,8 +9822,13 @@ } TARGET(POP_JUMP_IF_FALSE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_FALSE); _PyStackRef cond; @@ -8447,8 +9844,13 @@ } TARGET(POP_JUMP_IF_NONE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NONE); _PyStackRef value; @@ -8480,8 +9882,13 @@ } TARGET(POP_JUMP_IF_NOT_NONE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); _PyStackRef value; @@ -8513,8 +9920,13 @@ } TARGET(POP_JUMP_IF_TRUE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_TRUE); _PyStackRef cond; @@ -8530,6 +9942,10 @@ } TARGET(POP_TOP) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_TOP); @@ -8542,6 +9958,10 @@ } TARGET(PUSH_EXC_INFO) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_EXC_INFO); @@ -8567,6 +9987,10 @@ } TARGET(PUSH_NULL) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_NULL); @@ -8579,8 +10003,13 @@ } TARGET(RAISE_VARARGS) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RAISE_VARARGS); _PyStackRef *args; @@ -8598,14 +10027,19 @@ _PyFrame_SetStackPointer(frame, stack_pointer); monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } - goto error; + JUMP_TO_LABEL(error); } TARGET(RERAISE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RERAISE); _PyStackRef *values; @@ -8631,7 +10065,7 @@ _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); Py_DECREF(exc); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -8643,10 +10077,15 @@ _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); + DISPATCH(); } TARGET(RESERVED) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESERVED); @@ -8656,6 +10095,10 @@ } TARGET(RESUME) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME); @@ -8672,7 +10115,7 @@ _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); if (bytecode == NULL) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); @@ -8696,7 +10139,7 @@ int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } next_instr = this_instr; DISPATCH(); @@ -8721,7 +10164,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } } } @@ -8730,6 +10173,14 @@ } TARGET(RESUME_CHECK) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME_CHECK); @@ -8738,7 +10189,7 @@ if (_Py_emscripten_signal_clock == 0) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); - goto PREDICTED_RESUME; + JUMP_TO_PREDICTED(RESUME); } _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif @@ -8748,20 +10199,24 @@ if (eval_breaker != version) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); - goto PREDICTED_RESUME; + JUMP_TO_PREDICTED(RESUME); } #ifdef Py_GIL_DISABLED if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); - goto PREDICTED_RESUME; + JUMP_TO_PREDICTED(RESUME); } #endif DISPATCH(); } TARGET(RETURN_GENERATOR) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); @@ -8772,7 +10227,7 @@ PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); stack_pointer = _PyFrame_GetStackPointer(frame); if (gen == NULL) { - goto error; + JUMP_TO_LABEL(error); } assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -8797,6 +10252,10 @@ } TARGET(RETURN_VALUE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_VALUE); @@ -8825,6 +10284,10 @@ } TARGET(SEND) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND); @@ -8904,7 +10367,7 @@ } else { PyStackRef_CLOSE(v); - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } } stack_pointer += -1; @@ -8921,6 +10384,14 @@ } TARGET(SEND_GEN) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND_GEN); @@ -8935,7 +10406,7 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); - goto PREDICTED_SEND; + JUMP_TO_PREDICTED(SEND); } } // _SEND_GEN_FRAME @@ -8946,12 +10417,12 @@ if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); - goto PREDICTED_SEND; + JUMP_TO_PREDICTED(SEND); } if (gen->gi_frame_state >= FRAME_EXECUTING) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); - goto PREDICTED_SEND; + JUMP_TO_PREDICTED(SEND); } STAT_INC(SEND, hit); gen_frame = &gen->gi_iframe; @@ -8985,6 +10456,10 @@ } TARGET(SETUP_ANNOTATIONS) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SETUP_ANNOTATIONS); @@ -8994,21 +10469,21 @@ _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } /* check if __annotations__ in locals()... */ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } if (ann_dict == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); ann_dict = PyDict_New(); stack_pointer = _PyFrame_GetStackPointer(frame); if (ann_dict == NULL) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), @@ -9016,7 +10491,7 @@ Py_DECREF(ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } } else { @@ -9028,6 +10503,10 @@ } TARGET(SET_ADD) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_ADD); @@ -9041,7 +10520,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); if (err) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -9049,6 +10528,10 @@ } TARGET(SET_FUNCTION_ATTRIBUTE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); @@ -9073,6 +10556,10 @@ } TARGET(SET_UPDATE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_UPDATE); @@ -9086,7 +10573,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); if (err < 0) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -9094,6 +10581,10 @@ } TARGET(STORE_ATTR) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR); @@ -9132,7 +10623,7 @@ PyStackRef_CLOSE(v); PyStackRef_CLOSE(owner); if (err) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } } stack_pointer += -2; @@ -9141,8 +10632,13 @@ } TARGET(STORE_ATTR_INSTANCE_VALUE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); @@ -9158,7 +10654,7 @@ if (!LOCK_OBJECT(owner_o)) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - goto PREDICTED_STORE_ATTR; + JUMP_TO_PREDICTED(STORE_ATTR); } PyTypeObject *tp = Py_TYPE(owner_o); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { @@ -9166,7 +10662,7 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - goto PREDICTED_STORE_ATTR; + JUMP_TO_PREDICTED(STORE_ATTR); } } } @@ -9181,7 +10677,7 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - goto PREDICTED_STORE_ATTR; + JUMP_TO_PREDICTED(STORE_ATTR); } } } @@ -9212,8 +10708,13 @@ } TARGET(STORE_ATTR_SLOT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); @@ -9229,7 +10730,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - goto PREDICTED_STORE_ATTR; + JUMP_TO_PREDICTED(STORE_ATTR); } } // _STORE_ATTR_SLOT @@ -9240,7 +10741,7 @@ if (!LOCK_OBJECT(owner_o)) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - goto PREDICTED_STORE_ATTR; + JUMP_TO_PREDICTED(STORE_ATTR); } char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); @@ -9258,8 +10759,13 @@ } TARGET(STORE_ATTR_WITH_HINT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); @@ -9275,7 +10781,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - goto PREDICTED_STORE_ATTR; + JUMP_TO_PREDICTED(STORE_ATTR); } } // _STORE_ATTR_WITH_HINT @@ -9288,12 +10794,12 @@ if (dict == NULL) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - goto PREDICTED_STORE_ATTR; + JUMP_TO_PREDICTED(STORE_ATTR); } if (!LOCK_OBJECT(dict)) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - goto PREDICTED_STORE_ATTR; + JUMP_TO_PREDICTED(STORE_ATTR); } #ifdef Py_GIL_DISABLED if (dict != _PyObject_GetManagedDict(owner_o)) { @@ -9301,7 +10807,7 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - goto PREDICTED_STORE_ATTR; + JUMP_TO_PREDICTED(STORE_ATTR); } } #endif @@ -9313,7 +10819,7 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - goto PREDICTED_STORE_ATTR; + JUMP_TO_PREDICTED(STORE_ATTR); } } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; @@ -9322,7 +10828,7 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - goto PREDICTED_STORE_ATTR; + JUMP_TO_PREDICTED(STORE_ATTR); } } PyObject *old_value = ep->me_value; @@ -9331,7 +10837,7 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - goto PREDICTED_STORE_ATTR; + JUMP_TO_PREDICTED(STORE_ATTR); } } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9353,6 +10859,10 @@ } TARGET(STORE_DEREF) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_DEREF); @@ -9368,6 +10878,10 @@ } TARGET(STORE_FAST) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST); @@ -9380,6 +10894,10 @@ } TARGET(STORE_FAST_LOAD_FAST) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); @@ -9395,6 +10913,10 @@ } TARGET(STORE_FAST_STORE_FAST) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_STORE_FAST); @@ -9412,6 +10934,10 @@ } TARGET(STORE_GLOBAL) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_GLOBAL); @@ -9423,7 +10949,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); if (err) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -9431,6 +10957,10 @@ } TARGET(STORE_NAME) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_NAME); @@ -9445,7 +10975,7 @@ "no locals found when storing %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } if (PyDict_CheckExact(ns)) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9459,7 +10989,7 @@ } PyStackRef_CLOSE(v); if (err) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -9467,6 +10997,10 @@ } TARGET(STORE_SLICE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_SLICE); @@ -9508,7 +11042,7 @@ PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); if (err) { - goto pop_4_error; + JUMP_TO_LABEL(pop_4_error); } } stack_pointer += -4; @@ -9517,6 +11051,10 @@ } TARGET(STORE_SUBSCR) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR); @@ -9555,7 +11093,7 @@ PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); if (err) { - goto pop_3_error; + JUMP_TO_LABEL(pop_3_error); } } stack_pointer += -3; @@ -9564,6 +11102,14 @@ } TARGET(STORE_SUBSCR_DICT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_DICT); @@ -9579,7 +11125,7 @@ if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - goto PREDICTED_STORE_SUBSCR; + JUMP_TO_PREDICTED(STORE_SUBSCR); } STAT_INC(STORE_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9593,12 +11139,20 @@ PyStackRef_CLOSE(dict_st); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } DISPATCH(); } TARGET(STORE_SUBSCR_LIST_INT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); @@ -9615,24 +11169,24 @@ if (!PyLong_CheckExact(sub)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - goto PREDICTED_STORE_SUBSCR; + JUMP_TO_PREDICTED(STORE_SUBSCR); } if (!PyList_CheckExact(list)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - goto PREDICTED_STORE_SUBSCR; + JUMP_TO_PREDICTED(STORE_SUBSCR); } // Ensure nonnegative, zero-or-one-digit ints. if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - goto PREDICTED_STORE_SUBSCR; + JUMP_TO_PREDICTED(STORE_SUBSCR); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (!LOCK_OBJECT(list)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - goto PREDICTED_STORE_SUBSCR; + JUMP_TO_PREDICTED(STORE_SUBSCR); } // Ensure index < len(list) if (index >= PyList_GET_SIZE(list)) { @@ -9640,7 +11194,7 @@ if (true) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - goto PREDICTED_STORE_SUBSCR; + JUMP_TO_PREDICTED(STORE_SUBSCR); } } STAT_INC(STORE_SUBSCR, hit); @@ -9659,6 +11213,10 @@ } TARGET(SWAP) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SWAP); @@ -9674,6 +11232,10 @@ } TARGET(TO_BOOL) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL); @@ -9707,7 +11269,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (err < 0) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } res = err ? PyStackRef_True : PyStackRef_False; } @@ -9716,8 +11278,13 @@ } TARGET(TO_BOOL_ALWAYS_TRUE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); @@ -9734,7 +11301,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - goto PREDICTED_TO_BOOL; + JUMP_TO_PREDICTED(TO_BOOL); } } // _REPLACE_WITH_TRUE @@ -9748,6 +11315,14 @@ } TARGET(TO_BOOL_BOOL) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_BOOL); @@ -9759,13 +11334,21 @@ if (!PyStackRef_BoolCheck(value)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - goto PREDICTED_TO_BOOL; + JUMP_TO_PREDICTED(TO_BOOL); } STAT_INC(TO_BOOL, hit); DISPATCH(); } TARGET(TO_BOOL_INT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_INT); @@ -9779,7 +11362,7 @@ if (!PyLong_CheckExact(value_o)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - goto PREDICTED_TO_BOOL; + JUMP_TO_PREDICTED(TO_BOOL); } STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value_o)) { @@ -9795,6 +11378,14 @@ } TARGET(TO_BOOL_LIST) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_LIST); @@ -9808,7 +11399,7 @@ if (!PyList_CheckExact(value_o)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - goto PREDICTED_TO_BOOL; + JUMP_TO_PREDICTED(TO_BOOL); } STAT_INC(TO_BOOL, hit); res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; @@ -9818,6 +11409,14 @@ } TARGET(TO_BOOL_NONE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_NONE); @@ -9831,7 +11430,7 @@ if (!PyStackRef_IsNone(value)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - goto PREDICTED_TO_BOOL; + JUMP_TO_PREDICTED(TO_BOOL); } STAT_INC(TO_BOOL, hit); res = PyStackRef_False; @@ -9840,6 +11439,14 @@ } TARGET(TO_BOOL_STR) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_STR); @@ -9853,7 +11460,7 @@ if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - goto PREDICTED_TO_BOOL; + JUMP_TO_PREDICTED(TO_BOOL); } STAT_INC(TO_BOOL, hit); if (value_o == &_Py_STR(empty)) { @@ -9870,6 +11477,10 @@ } TARGET(UNARY_INVERT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_INVERT); @@ -9881,7 +11492,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (res_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; @@ -9889,6 +11500,10 @@ } TARGET(UNARY_NEGATIVE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NEGATIVE); @@ -9900,7 +11515,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (res_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; @@ -9908,6 +11523,10 @@ } TARGET(UNARY_NOT) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NOT); @@ -9922,6 +11541,10 @@ } TARGET(UNPACK_EX) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNPACK_EX); @@ -9935,7 +11558,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); if (res == 0) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } stack_pointer += (oparg & 0xFF) + (oparg >> 8); assert(WITHIN_STACK_BOUNDS()); @@ -9943,6 +11566,10 @@ } TARGET(UNPACK_SEQUENCE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE); @@ -9979,7 +11606,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); if (res == 0) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } } stack_pointer += -1 + oparg; @@ -9988,6 +11615,14 @@ } TARGET(UNPACK_SEQUENCE_LIST) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); @@ -10001,19 +11636,19 @@ if (!PyList_CheckExact(seq_o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - goto PREDICTED_UNPACK_SEQUENCE; + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } if (!LOCK_OBJECT(seq_o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - goto PREDICTED_UNPACK_SEQUENCE; + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } if (PyList_GET_SIZE(seq_o) != oparg) { UNLOCK_OBJECT(seq_o); if (true) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - goto PREDICTED_UNPACK_SEQUENCE; + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } } STAT_INC(UNPACK_SEQUENCE, hit); @@ -10029,6 +11664,14 @@ } TARGET(UNPACK_SEQUENCE_TUPLE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); @@ -10042,12 +11685,12 @@ if (!PyTuple_CheckExact(seq_o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - goto PREDICTED_UNPACK_SEQUENCE; + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } if (PyTuple_GET_SIZE(seq_o) != oparg) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - goto PREDICTED_UNPACK_SEQUENCE; + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq_o); @@ -10061,6 +11704,14 @@ } TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ + #ifdef Py_TAIL_CALL_INTERP + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); @@ -10075,12 +11726,12 @@ if (!PyTuple_CheckExact(seq_o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - goto PREDICTED_UNPACK_SEQUENCE; + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } if (PyTuple_GET_SIZE(seq_o) != 2) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - goto PREDICTED_UNPACK_SEQUENCE; + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); @@ -10094,6 +11745,10 @@ } TARGET(WITH_EXCEPT_START) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(WITH_EXCEPT_START); @@ -10138,7 +11793,7 @@ (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; @@ -10148,6 +11803,10 @@ } TARGET(YIELD_VALUE) { + #ifdef Py_TAIL_CALL_INTERP + int opcode = next_instr->op.code; + (void)(opcode); + #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(YIELD_VALUE); @@ -10194,6 +11853,7 @@ } /* END INSTRUCTIONS */ +#ifndef Py_TAIL_CALL_INTERP #if USE_COMPUTED_GOTOS _unknown_opcode: #else @@ -10202,12 +11862,14 @@ /* Tell C compilers not to hold the opcode variable in the loop. next_instr points the current instruction without TARGET(). */ opcode = next_instr->op.code; + _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", _PyFrame_GetCode(frame)->co_filename, PyUnstable_InterpreterFrame_GetLine(frame), opcode); - goto error; + JUMP_TO_LABEL(error); + } @@ -10217,31 +11879,31 @@ #endif /* Py_TAIL_CALL_INTERP */ /* BEGIN LABELS */ - pop_4_error: + LABEL(pop_4_error) { STACK_SHRINK(4); - goto error; + JUMP_TO_LABEL(error); } - pop_3_error: + LABEL(pop_3_error) { STACK_SHRINK(3); - goto error; + JUMP_TO_LABEL(error); } - pop_2_error: + LABEL(pop_2_error) { STACK_SHRINK(2); - goto error; + JUMP_TO_LABEL(error); } - pop_1_error: + LABEL(pop_1_error) { STACK_SHRINK(1); - goto error; + JUMP_TO_LABEL(error); } - error: + LABEL(error) { /* Double-check exception status. */ #ifdef NDEBUG @@ -10262,10 +11924,10 @@ } } _PyEval_MonitorRaise(tstate, frame, next_instr-1); - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } - exception_unwind: + LABEL(exception_unwind) { /* We can't use frame->instr_ptr here, as RERAISE may have set it */ int offset = INSTR_OFFSET()-1; @@ -10281,7 +11943,7 @@ assert(STACK_LEVEL() == 0); _PyFrame_SetStackPointer(frame, stack_pointer); monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; + JUMP_TO_LABEL(exit_unwind); } assert(STACK_LEVEL() >= level); _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; @@ -10292,7 +11954,7 @@ int frame_lasti = _PyInterpreterFrame_LASTI(frame); PyObject *lasti = PyLong_FromLong(frame_lasti); if (lasti == NULL) { - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } PUSH(PyStackRef_FromPyObjectSteal(lasti)); } @@ -10304,7 +11966,7 @@ PUSH(PyStackRef_FromPyObjectSteal(exc)); next_instr = _PyFrame_GetBytecode(frame) + handler; if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } /* Resume normal execution */ #ifdef LLTRACE @@ -10312,10 +11974,14 @@ lltrace_resume_frame(frame); } #endif + #ifdef Py_TAIL_CALL_INTERP + int opcode; + (void)(opcode); + #endif DISPATCH(); } - exit_unwind: + LABEL(exit_unwind) { assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); @@ -10333,13 +11999,13 @@ } next_instr = frame->instr_ptr; stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } - start_frame: + LABEL(start_frame) { if (_Py_EnterRecursivePy(tstate)) { - goto exit_unwind; + JUMP_TO_LABEL(exit_unwind); } next_instr = frame->instr_ptr; stack_pointer = _PyFrame_GetStackPointer(frame); @@ -10348,7 +12014,7 @@ int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); frame->lltrace = lltrace; if (lltrace < 0) { - goto exit_unwind; + JUMP_TO_LABEL(exit_unwind); } } #endif @@ -10360,6 +12026,10 @@ assert(!_PyErr_Occurred(tstate)); #endif + #ifdef Py_TAIL_CALL_INTERP + int opcode; + (void)(opcode); + #endif DISPATCH(); } diff --git a/Python/generated_tail_call_handlers.c.h b/Python/generated_tail_call_handlers.c.h deleted file mode 100644 index bcee0e6b9ca525..00000000000000 --- a/Python/generated_tail_call_handlers.c.h +++ /dev/null @@ -1,15107 +0,0 @@ -// This file is generated by Tools/cases_generator/tier1_tail_call_generator.py -// from: -// Python/bytecodes.c -// Do not edit! - -#ifndef Py_TAIL_CALL_INTERP - #error "This file is for tail-calling interpreter only." -#endif -#define TIER_ONE 1 -static inline PyObject *_TAIL_CALL_entry(TAIL_CALL_PARAMS); -static py_tail_call_funcptr INSTRUCTION_TABLE[256]; - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_4_error(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_3_error(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); - -/* BEGIN INSTRUCTIONS */ -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_4_error(TAIL_CALL_PARAMS) -{ - STACK_SHRINK(4); - TAIL_CALL(error); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_3_error(TAIL_CALL_PARAMS) -{ - STACK_SHRINK(3); - TAIL_CALL(error); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS) -{ - STACK_SHRINK(2); - TAIL_CALL(error); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS) -{ - STACK_SHRINK(1); - TAIL_CALL(error); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS) -{ - /* Double-check exception status. */ - #ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } - #else - assert(_PyErr_Occurred(tstate)); - #endif - - /* Log traceback info. */ - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); - TAIL_CALL(exception_unwind); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS) -{ - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - TAIL_CALL(exit_unwind); - } - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - TAIL_CALL(exception_unwind); - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - TAIL_CALL(exception_unwind); - } - /* Resume normal execution */ - #ifdef LLTRACE - if (frame->lltrace >= 5) { - lltrace_resume_frame(frame); - } - #endif - DISPATCH(); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS) -{ - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - TAIL_CALL(error); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS) -{ - if (_Py_EnterRecursivePy(tstate)) { - TAIL_CALL(exit_unwind); - } - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - #ifdef LLTRACE - { - int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); - frame->lltrace = lltrace; - if (lltrace < 0) { - TAIL_CALL(exit_unwind); - } - } - #endif - - #ifdef Py_DEBUG - /* _PyEval_EvalFrameDefault() must not be called with an exception set, - because it can clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!_PyErr_Occurred(tstate)); - #endif - - DISPATCH(); -} - - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP); - PREDICTED_BINARY_OP:; - _Py_CODEUNIT* const this_instr = next_instr - 6; - (void)this_instr; - _PyStackRef lhs; - _PyStackRef rhs; - _PyStackRef res; - // _SPECIALIZE_BINARY_OP - { - rhs = stack_pointer[-1]; - lhs = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(BINARY_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - assert(NB_ADD <= oparg); - assert(oparg <= NB_INPLACE_XOR); - } - /* Skip 4 cache entries */ - // _BINARY_OP - { - PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); - PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); - assert(_PyEval_BinaryOps[oparg]); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(lhs); - PyStackRef_CLOSE(rhs); - if (res_o == NULL) { - goto pop_2_error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_FLOAT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!PyFloat_CheckExact(right_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_ADD_FLOAT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval + - ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) { - goto pop_2_error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_INT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyLong_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!PyLong_CheckExact(right_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_ADD_INT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) { - goto pop_2_error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_UNICODE - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyUnicode_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!PyUnicode_CheckExact(right_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_ADD_UNICODE - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyUnicode_Concat(left_o, right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - if (res_o == NULL) { - goto pop_2_error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_EXTEND); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - /* Skip 1 cache entry */ - // _GUARD_BINARY_OP_EXTEND - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *descr = read_obj(&this_instr[2].cache); - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; - assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); - assert(d && d->guard); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = d->guard(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!res) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - } - /* Skip -4 cache entry */ - // _BINARY_OP_EXTEND - { - PyObject *descr = read_obj(&this_instr[2].cache); - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); - _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; - STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = d->action(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - // _GUARD_BOTH_UNICODE - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyUnicode_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!PyUnicode_CheckExact(right_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_INPLACE_ADD_UNICODE - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(right_o)); - int next_oparg; - #if TIER_ONE - assert(next_instr->op.code == STORE_FAST); - next_oparg = next_instr->op.arg; - #else - next_oparg = CURRENT_OPERAND0(); - #endif - _PyStackRef *target_local = &GETLOCAL(next_oparg); - if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(BINARY_OP, hit); - /* Handle `left = left + right` or `left += right` for str. - * - * When possible, extend `left` in place rather than - * allocating a new PyUnicodeObject. This attempts to avoid - * quadratic behavior when one neglects to use str.join(). - * - * If `left` has only two references remaining (one from - * the stack, one in the locals), DECREFing `left` leaves - * only the locals reference, so PyUnicode_Append knows - * that the string is safe to mutate. - */ - assert(Py_REFCNT(left_o) >= 2); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); - PyUnicode_Append(&temp, right_o); - *target_local = PyStackRef_FromPyObjectSteal(temp); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (PyStackRef_IsNull(*target_local)) { - goto pop_2_error; - } - #if TIER_ONE - // The STORE_FAST is already done. This is done here in tier one, - // and during trace projection in tier two: - assert(next_instr->op.code == STORE_FAST); - SKIP_OVER(1); - #endif - } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_FLOAT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!PyFloat_CheckExact(right_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_MULTIPLY_FLOAT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval * - ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) { - goto pop_2_error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_INT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyLong_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!PyLong_CheckExact(right_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_MULTIPLY_INT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) { - goto pop_2_error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_FLOAT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!PyFloat_CheckExact(right_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBTRACT_FLOAT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval - - ((PyFloatObject *)right_o)->ob_fval; - PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (res_o == NULL) { - goto pop_2_error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_INT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyLong_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!PyLong_CheckExact(right_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBTRACT_INT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) { - goto pop_2_error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BINARY_SLICE); - _PyStackRef container; - _PyStackRef start; - _PyStackRef stop; - _PyStackRef res; - // _SPECIALIZE_BINARY_SLICE - { - // Placeholder until we implement BINARY_SLICE specialization - #if ENABLE_SPECIALIZATION - OPCODE_DEFERRED_INC(BINARY_SLICE); - #endif /* ENABLE_SPECIALIZATION */ - } - // _BINARY_SLICE - { - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyObject *res_o; - // Can't use ERROR_IF() here, because we haven't - // DECREF'ed container yet, and we still own slice. - if (slice == NULL) { - res_o = NULL; - } - else { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); - Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - } - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(container); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR); - PREDICTED_BINARY_SUBSCR:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef container; - _PyStackRef sub; - _PyStackRef res; - // _SPECIALIZE_BINARY_SUBSCR - { - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - assert(frame->stackpointer == NULL); - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_BinarySubscr(container, sub, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(BINARY_SUBSCR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _BINARY_SUBSCR - { - PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); - PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_GetItem(container_o, sub_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (res_o == NULL) { - goto pop_2_error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_DICT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef dict_st; - _PyStackRef sub_st; - _PyStackRef res; - /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - dict_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(BINARY_SUBSCR, hit); - PyObject *res_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int rc = PyDict_GetItemRef(dict, sub, &res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (rc == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetKeyError(sub); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - PyStackRef_CLOSE(dict_st); - PyStackRef_CLOSE(sub_st); - if (rc <= 0) { - goto pop_2_error; - } - // not found or error - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef container; - _PyStackRef getitem; - _PyStackRef sub; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _BINARY_SUBSCR_CHECK_FUNC - { - container = stack_pointer[-2]; - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - if (getitem_o == NULL) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - assert(PyFunction_Check(getitem_o)); - uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); - assert(code->co_argcount == 2); - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - getitem = PyStackRef_FromPyObjectNew(getitem_o); - STAT_INC(BINARY_SUBSCR, hit); - } - // _BINARY_SUBSCR_INIT_CALL - { - sub = stack_pointer[-1]; - new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); - new_frame->localsplus[0] = container; - new_frame->localsplus[1] = sub; - frame->return_offset = 2 ; - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef list_st; - _PyStackRef sub_st; - _PyStackRef res; - /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - if (!PyLong_CheckExact(sub)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!PyList_CheckExact(list)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - // Deopt unless 0 <= sub < PyList_Size(list) - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - #ifdef Py_GIL_DISABLED - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(BINARY_SUBSCR, hit); - #else - if (index >= PyList_GET_SIZE(list)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(BINARY_SUBSCR, hit); - PyObject *res_o = PyList_GET_ITEM(list, index); - assert(res_o != NULL); - Py_INCREF(res_o); - #endif - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(list_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef str_st; - _PyStackRef sub_st; - _PyStackRef res; - /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - str_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - if (!PyLong_CheckExact(sub)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!PyUnicode_CheckExact(str)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - // Specialize for reading an ASCII character from any string: - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(BINARY_SUBSCR, hit); - PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(str_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef tuple_st; - _PyStackRef sub_st; - _PyStackRef res; - /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - tuple_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - if (!PyLong_CheckExact(sub)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!PyTuple_CheckExact(tuple)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - // Deopt unless 0 <= sub < PyTuple_Size(list) - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) { - UPDATE_MISS_STATS(BINARY_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_BINARY_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(BINARY_SUBSCR, hit); - PyObject *res_o = PyTuple_GET_ITEM(tuple, index); - assert(res_o != NULL); - Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(tuple_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_LIST); - _PyStackRef *values; - _PyStackRef list; - values = &stack_pointer[-oparg]; - PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); - if (list_o == NULL) { - goto error; - } - list = PyStackRef_FromPyObjectSteal(list_o); - stack_pointer[-oparg] = list; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_MAP); - _PyStackRef *values; - _PyStackRef map; - values = &stack_pointer[-oparg*2]; - STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); - if (CONVERSION_FAILED(values_o)) { - for (int _i = oparg*2; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); - } - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *map_o = _PyDict_FromItems( - values_o, 2, - values_o+1, 2, - oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); - for (int _i = oparg*2; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); - } - if (map_o == NULL) { - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - map = PyStackRef_FromPyObjectSteal(map_o); - stack_pointer[-oparg*2] = map; - stack_pointer += 1 - oparg*2; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SET); - _PyStackRef *values; - _PyStackRef set; - values = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *set_o = PySet_New(NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (set_o == NULL) { - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); - } - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - int err = 0; - for (int i = 0; i < oparg; i++) { - if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PySet_Add(set_o, PyStackRef_AsPyObjectBorrow(values[i])); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); - } - if (err != 0) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(set_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - set = PyStackRef_FromPyObjectSteal(set_o); - stack_pointer[-oparg] = set; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SLICE); - _PyStackRef *args; - _PyStackRef slice; - args = &stack_pointer[-oparg]; - PyObject *start_o = PyStackRef_AsPyObjectBorrow(args[0]); - PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); - PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; - PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - if (slice_o == NULL) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - slice = PyStackRef_FromPyObjectSteal(slice_o); - stack_pointer[-oparg] = slice; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_STRING); - _PyStackRef *pieces; - _PyStackRef str; - pieces = &stack_pointer[-oparg]; - STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); - if (CONVERSION_FAILED(pieces_o)) { - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(pieces[_i]); - } - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(pieces[_i]); - } - if (str_o == NULL) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - str = PyStackRef_FromPyObjectSteal(str_o); - stack_pointer[-oparg] = str; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_TUPLE); - _PyStackRef *values; - _PyStackRef tup; - values = &stack_pointer[-oparg]; - PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); - if (tup_o == NULL) { - goto error; - } - tup = PyStackRef_FromPyObjectSteal(tup_o); - stack_pointer[-oparg] = tup; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CACHE); - assert(0 && "Executing a cache."); - Py_FatalError("Executing a cache."); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL); - PREDICTED_CALL:; - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - opcode = CALL; - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef *func; - _PyStackRef *maybe_self; - _PyStackRef res; - // _SPECIALIZE_CALL - { - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CALL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD - { - args = &stack_pointer[-oparg]; - func = &stack_pointer[-2 - oparg]; - maybe_self = &stack_pointer[-1 - oparg]; - if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _DO_CALL - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - // Check if the call can be inlined or not - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - arguments, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - goto error; - } - frame->return_offset = 4 ; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(res_o); - } - } - } - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - goto error; - } - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *null; - _PyStackRef *args; - _PyStackRef *init; - _PyStackRef *self; - _PyInterpreterFrame *init_frame; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _CHECK_AND_ALLOCATE_OBJECT - { - args = &stack_pointer[-oparg]; - null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - init = &stack_pointer[-2 - oparg]; - self = &stack_pointer[-1 - oparg]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - if (!PyStackRef_IsNull(null[0])) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!PyType_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - PyTypeObject *tp = (PyTypeObject *)callable_o; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - assert(tp->tp_new == PyBaseObject_Type.tp_new); - assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); - assert(tp->tp_alloc == PyType_GenericAlloc); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; - PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); - PyCodeObject *code = (PyCodeObject *)init_func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *self_o = PyType_GenericAlloc(tp, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (self_o == NULL) { - goto error; - } - self[0] = PyStackRef_FromPyObjectSteal(self_o); - _PyStackRef temp = callable[0]; - init[0] = PyStackRef_FromPyObjectNew(init_func); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // _CREATE_INIT_FRAME - { - args = &stack_pointer[-oparg]; - self = &stack_pointer[-1 - oparg]; - init = &stack_pointer[-2 - oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); - assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); - assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); - stack_pointer = _PyFrame_GetStackPointer(frame); - /* Push self onto stack of shim */ - shim->localsplus[0] = PyStackRef_DUP(self[0]); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, init[0], NULL, args-1, oparg+1, NULL, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - _PyEval_FrameClearAndPop(tstate, shim); - goto error; - } - init_frame = temp; - frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; - /* Account for pushing the extra frame. - * We don't check recursion depth here, - * as it will be checked after start_frame */ - tstate->py_recursion_remaining--; - } - // _PUSH_FRAME - { - new_frame = init_frame; - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *null; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS - { - null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - if (!PyStackRef_IsNull(null[0])) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _INIT_CALL_BOUND_METHOD_EXACT_ARGS - { - self_or_null = null; - assert(PyStackRef_IsNull(self_or_null[0])); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - STAT_INC(CALL, hit); - self_or_null[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - _PyStackRef temp = callable[0]; - callable[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // flush - // _CHECK_FUNCTION_VERSION - { - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _CHECK_FUNCTION_EXACT_ARGS - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - assert(PyFunction_Check(callable_o)); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _CHECK_STACK_SPACE - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _INIT_CALL_PY_EXACT_ARGS - { - args = &stack_pointer[-oparg]; - int has_self = !PyStackRef_IsNull(self_or_null[0]); - STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null[0]; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; - } - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *null; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _CHECK_METHOD_VERSION - { - null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - if (Py_TYPE(callable_o) != &PyMethod_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - PyObject *func = ((PyMethodObject *)callable_o)->im_func; - if (!PyFunction_Check(func)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - if (((PyFunctionObject *)func)->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!PyStackRef_IsNull(null[0])) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _EXPAND_METHOD - { - self_or_null = null; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - assert(PyStackRef_IsNull(self_or_null[0])); - assert(Py_TYPE(callable_o) == &PyMethod_Type); - self_or_null[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - _PyStackRef temp = callable[0]; - callable[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(callable[0])); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // flush - // _PY_FRAME_GENERAL - { - args = &stack_pointer[-oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - args, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - // The frame has stolen all the arguments from the stack. - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - goto error; - } - new_frame = temp; - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_CLASS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_CLASS - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - if (!PyType_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - PyTypeObject *tp = (PyTypeObject *)callable_o; - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - if (tp->tp_vectorcall == NULL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - goto error; - } - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL functions, without keywords */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - if (!PyCFunction_CheckExact(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - /* res = func(self, args, nargs) */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable_o), - args_o, - total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - goto error; - } - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST_WITH_KEYWORDS - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - if (!PyCFunction_CheckExact(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(CALL, hit); - /* res = func(self, arguments, nargs, kwnames) */ - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - goto error; - } - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_O); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_O - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - /* Builtin METH_O functions */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - if (total_args != 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!PyCFunction_CheckExact(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - // CPython promises to check all non-vectorcall function calls. - if (tstate->c_recursion_remaining <= 0) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - _PyStackRef arg = args[0]; - _Py_EnterRecursiveCallTstateUnchecked(tstate); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable[0]); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - goto error; - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_FUNCTION_EX); - opcode = CALL_FUNCTION_EX; - _PyStackRef func; - _PyStackRef callargs; - _PyStackRef kwargs_in; - _PyStackRef tuple; - _PyStackRef kwargs_out; - _PyStackRef func_st; - _PyStackRef null; - _PyStackRef callargs_st; - _PyStackRef kwargs_st; - _PyStackRef result; - // _MAKE_CALLARGS_A_TUPLE - { - kwargs_in = stack_pointer[-1]; - callargs = stack_pointer[-2]; - func = stack_pointer[-4]; - PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); - if (PyTuple_CheckExact(callargs_o)) { - tuple = callargs; - kwargs_out = kwargs_in; - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (tuple_o == NULL) { - goto error; - } - kwargs_out = kwargs_in; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - tuple = PyStackRef_FromPyObjectSteal(tuple_o); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - } - } - // _DO_CALL_FUNCTION_EX - { - kwargs_st = kwargs_out; - callargs_st = tuple; - null = stack_pointer[-3]; - func_st = func; - (void)null; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - PyObject *result_o; - assert(!_PyErr_Occurred(tstate)); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - assert(PyTuple_CheckExact(callargs)); - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - stack_pointer[-2] = callargs_st; - stack_pointer[-1] = kwargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(result_o); - } - } - } - } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( - tstate, func_st, locals, - nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - // Need to sync the stack since we exit with DISPATCH_INLINED. - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { - goto error; - } - assert( 1 == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); - } - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[-2] = callargs_st; - stack_pointer[-1] = kwargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result_o == NULL) { - goto error; - } - result = PyStackRef_FromPyObjectSteal(result_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - goto error; - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_1); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - assert(oparg <= MAX_INTRINSIC_1); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (res_o == NULL) { - goto pop_1_error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_2); - _PyStackRef value2_st; - _PyStackRef value1_st; - _PyStackRef res; - value1_st = stack_pointer[-1]; - value2_st = stack_pointer[-2]; - assert(oparg <= MAX_INTRINSIC_2); - PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); - PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value2_st); - PyStackRef_CLOSE(value1_st); - if (res_o == NULL) { - goto pop_2_error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ISINSTANCE); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - /* isinstance(o, o2) */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - if (total_args != 2) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.isinstance) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(CALL, hit); - _PyStackRef cls_stackref = arguments[1]; - _PyStackRef inst_stackref = arguments[0]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (retval < 0) { - goto error; - } - res = retval ? PyStackRef_True : PyStackRef_False; - assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW); - PREDICTED_CALL_KW:; - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - opcode = CALL_KW; - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef kwnames_in; - _PyStackRef *func; - _PyStackRef *maybe_self; - _PyStackRef kwnames_out; - _PyStackRef res; - // _SPECIALIZE_CALL_KW - { - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CALL_KW); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD_KW - { - kwnames_in = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - func = &stack_pointer[-3 - oparg]; - maybe_self = &stack_pointer[-2 - oparg]; - if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - kwnames_out = kwnames_in; - } - // _DO_CALL_KW - { - kwnames = kwnames_out; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - // Check if the call can be inlined or not - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-1] = kwnames; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - // Sync stack explicitly since we leave using DISPATCH_INLINED(). - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - goto error; - } - assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); - frame->return_offset = 4 ; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - PyStackRef_CLOSE(kwnames); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - stack_pointer[-1] = kwnames; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(res_o); - } - } - } - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - PyStackRef_CLOSE(kwnames); - if (res_o == NULL) { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *null; - _PyStackRef kwnames; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _CHECK_METHOD_VERSION_KW - { - null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - if (Py_TYPE(callable_o) != &PyMethod_Type) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, oparg); - } - PyObject *func = ((PyMethodObject *)callable_o)->im_func; - if (!PyFunction_Check(func)) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, oparg); - } - if (((PyFunctionObject *)func)->func_version != func_version) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!PyStackRef_IsNull(null[0])) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _EXPAND_METHOD_KW - { - self_or_null = null; - assert(PyStackRef_IsNull(self_or_null[0])); - _PyStackRef callable_s = callable[0]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable_s); - assert(Py_TYPE(callable_o) == &PyMethod_Type); - self_or_null[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - callable[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(callable[0])); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable_s); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // flush - // _PY_FRAME_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - goto error; - } - new_frame = temp; - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_NON_PY); - opcode = CALL_KW_NON_PY; - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef kwnames; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CHECK_IS_NOT_PY_CALLABLE_KW - { - callable = &stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - if (PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, oparg); - } - if (Py_TYPE(callable_o) == &PyMethod_Type) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _CALL_KW_NON_PY - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - #if TIER_ONE - assert(opcode != INSTRUMENTED_CALL); - #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - PyStackRef_CLOSE(kwnames); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - goto error; - } - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_PY); - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef kwnames; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _CHECK_FUNCTION_VERSION_KW - { - callable = &stack_pointer[-3 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, oparg); - } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - Py_MUSTTAIL return _TAIL_CALL_CALL_KW(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _PY_FRAME_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - goto error; - } - new_frame = temp; - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LEN); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - /* len(o) */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - if (total_args != 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.len) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(CALL, hit); - _PyStackRef arg_stackref = args[0]; - PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_ssize_t len_i = PyObject_Length(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) { - goto error; - } - PyObject *res_o = PyLong_FromSsize_t(len_i); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res_o == NULL) { - GOTO_ERROR(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg_stackref); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable[0]); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LIST_APPEND); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self; - _PyStackRef arg; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - arg = stack_pointer[-1]; - self = stack_pointer[-2]; - callable = stack_pointer[-3]; - assert(oparg == 1); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.list_append) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - assert(self_o != NULL); - if (!PyList_Check(self_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!LOCK_OBJECT(self_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(CALL, hit); - int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); - UNLOCK_OBJECT(self_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; - } - #if TIER_ONE - // Skip the following POP_TOP. This is done here in tier one, and - // during trace projection in tier two: - assert(next_instr->op.code == POP_TOP); - SKIP_OVER(1); - #endif - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - /* Builtin METH_FASTCALL methods, without keywords */ - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_FASTCALL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - if (!Py_IS_TYPE(self, method->d_common.d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFast cfunc = - (PyCFunctionFast)(void(*)(void))meth->ml_meth; - PyObject *res_o = cfunc(self, (args_o + 1), nargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - goto error; - } - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - if (!Py_IS_TYPE(self, d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - goto error; - } - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_NOARGS - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - if (total_args != 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - PyMethodDef *meth = method->d_method; - _PyStackRef self_stackref = args[0]; - PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - if (!Py_IS_TYPE(self, method->d_common.d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - if (meth->ml_flags != METH_NOARGS) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - // CPython promises to check all non-vectorcall function calls. - if (tstate->c_recursion_remaining <= 0) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - _Py_EnterRecursiveCallTstateUnchecked(tstate); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self_stackref); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable[0]); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - goto error; - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_O - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (total_args != 2) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_O) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - // CPython promises to check all non-vectorcall function calls. - if (tstate->c_recursion_remaining <= 0) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - _PyStackRef arg_stackref = arguments[1]; - _PyStackRef self_stackref = arguments[0]; - if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - _Py_EnterRecursiveCallTstateUnchecked(tstate); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, - PyStackRef_AsPyObjectBorrow(self_stackref), - PyStackRef_AsPyObjectBorrow(arg_stackref)); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - goto error; - } - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_NON_PY_GENERAL); - opcode = CALL_NON_PY_GENERAL; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CHECK_IS_NOT_PY_CALLABLE - { - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - if (PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - if (Py_TYPE(callable_o) == &PyMethod_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _CALL_NON_PY_GENERAL - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - #if TIER_ONE - assert(opcode != INSTRUMENTED_CALL); - #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - goto error; - } - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _CHECK_FUNCTION_VERSION - { - callable = &stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _CHECK_FUNCTION_EXACT_ARGS - { - self_or_null = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - assert(PyFunction_Check(callable_o)); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _CHECK_STACK_SPACE - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _INIT_CALL_PY_EXACT_ARGS - { - args = &stack_pointer[-oparg]; - int has_self = !PyStackRef_IsNull(self_or_null[0]); - STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null[0]; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; - } - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_GENERAL); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _CHECK_FUNCTION_VERSION - { - callable = &stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _PY_FRAME_GENERAL - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null[0])) { - args--; - total_args++; - } - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - args, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - // The frame has stolen all the arguments from the stack. - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - goto error; - } - new_frame = temp; - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_STR_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_STR_1 - { - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - if (callable_o != (PyObject *)&PyUnicode_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Str(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - goto error; - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TUPLE_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_TUPLE_1 - { - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - if (callable_o != (PyObject *)&PyTuple_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PySequence_Tuple(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - goto error; - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TYPE_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - if (callable_o != (PyObject *)&PyType_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - Py_MUSTTAIL return _TAIL_CALL_CALL(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(CALL, hit); - res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); - stack_pointer[-3] = res; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EG_MATCH); - _PyStackRef exc_value_st; - _PyStackRef match_type_st; - _PyStackRef rest; - _PyStackRef match; - match_type_st = stack_pointer[-1]; - exc_value_st = stack_pointer[-2]; - PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - PyStackRef_CLOSE(exc_value_st); - PyStackRef_CLOSE(match_type_st); - goto pop_2_error; - } - PyObject *match_o = NULL; - PyObject *rest_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, - &match_o, &rest_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(exc_value_st); - PyStackRef_CLOSE(match_type_st); - if (res < 0) { - goto pop_2_error; - } - assert((match_o == NULL) == (rest_o == NULL)); - if (match_o == NULL) { - goto pop_2_error; - } - if (!Py_IsNone(match_o)) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyErr_SetHandledException(match_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - } - rest = PyStackRef_FromPyObjectSteal(rest_o); - match = PyStackRef_FromPyObjectSteal(match_o); - stack_pointer[-2] = rest; - stack_pointer[-1] = match; - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EXC_MATCH); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyExceptionInstance_Check(left_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyEval_CheckExceptTypeValid(tstate, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - PyStackRef_CLOSE(right); - goto pop_1_error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PyErr_GivenExceptionMatches(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(right); - b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[-1] = b; - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(CLEANUP_THROW); - _PyStackRef sub_iter_st; - _PyStackRef last_sent_val_st; - _PyStackRef exc_value_st; - _PyStackRef none; - _PyStackRef value; - exc_value_st = stack_pointer[-1]; - last_sent_val_st = stack_pointer[-2]; - sub_iter_st = stack_pointer[-3]; - PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - #ifndef Py_TAIL_CALL_INTERP - assert(throwflag); - #endif - assert(exc_value && PyExceptionInstance_Check(exc_value)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - none = PyStackRef_None; - value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); - PyStackRef_CLOSE(sub_iter_st); - PyStackRef_CLOSE(last_sent_val_st); - PyStackRef_CLOSE(exc_value_st); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); - monitor_reraise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto exception_unwind; - } - stack_pointer[-3] = none; - stack_pointer[-2] = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP); - PREDICTED_COMPARE_OP:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _SPECIALIZE_COMPARE_OP - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(COMPARE_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _COMPARE_OP - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert((oparg >> 5) <= Py_GE); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - if (res_o == NULL) { - goto pop_2_error; - } - if (oparg & 16) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res_bool = PyObject_IsTrue(res_o); - Py_DECREF(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_bool < 0) { - goto error; - } - res = res_bool ? PyStackRef_True : PyStackRef_False; - } - else { - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_FLOAT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!PyFloat_CheckExact(right_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - } - /* Skip 1 cache entry */ - // _COMPARE_OP_FLOAT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left_o); - double dright = PyFloat_AS_DOUBLE(right_o); - // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg - int sign_ish = COMPARISON_BIT(dleft, dright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - // It's always a bool, so we don't care about oparg & 16. - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_INT); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_INT - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyLong_CheckExact(left_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!PyLong_CheckExact(right_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - } - /* Skip 1 cache entry */ - // _COMPARE_OP_INT - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!_PyLong_IsCompact((PyLongObject *)left_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!_PyLong_IsCompact((PyLongObject *)right_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && - _PyLong_DigitCount((PyLongObject *)right_o) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); - // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg - int sign_ish = COMPARISON_BIT(ileft, iright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - // It's always a bool, so we don't care about oparg & 16. - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_STR); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_BOTH_UNICODE - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyUnicode_CheckExact(left_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!PyUnicode_CheckExact(right_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - Py_MUSTTAIL return _TAIL_CALL_COMPARE_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - } - /* Skip 1 cache entry */ - // _COMPARE_OP_STR - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left_o, right_o); - assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(eq == 0 || eq == 1); - assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); - assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; - // It's always a bool, so we don't care about oparg & 16. - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP); - PREDICTED_CONTAINS_OP:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - // _SPECIALIZE_CONTAINS_OP - { - right = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ContainsOp(right, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CONTAINS_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _CONTAINS_OP - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PySequence_Contains(right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - if (res < 0) { - goto pop_2_error; - } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[-2] = b; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP_DICT); - static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - /* Skip 1 cache entry */ - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyDict_CheckExact(right_o)) { - UPDATE_MISS_STATS(CONTAINS_OP); - assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); - Py_MUSTTAIL return _TAIL_CALL_CONTAINS_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(CONTAINS_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PyDict_Contains(right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - if (res < 0) { - goto pop_2_error; - } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP_SET); - static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - /* Skip 1 cache entry */ - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o))) { - UPDATE_MISS_STATS(CONTAINS_OP); - assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); - Py_MUSTTAIL return _TAIL_CALL_CONTAINS_OP(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(CONTAINS_OP, hit); - // Note: both set and frozenset use the same seq_contains method! - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PySet_Contains((PySetObject *)right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - if (res < 0) { - goto pop_2_error; - } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CONVERT_VALUE); - _PyStackRef value; - _PyStackRef result; - value = stack_pointer[-1]; - conversion_func conv_fn; - assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = _PyEval_ConversionFuncs[oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result_o == NULL) { - goto error; - } - result = PyStackRef_FromPyObjectSteal(result_o); - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY); - _PyStackRef bottom; - _PyStackRef top; - bottom = stack_pointer[-1 - (oparg-1)]; - assert(oparg > 0); - top = PyStackRef_DUP(bottom); - stack_pointer[0] = top; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY_FREE_VARS); - /* Copy closure variables to free variables */ - PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - PyObject *closure = func->func_closure; - assert(oparg == co->co_nfreevars); - int offset = co->co_nlocalsplus - oparg; - for (int i = 0; i < oparg; ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_ATTR); - _PyStackRef owner; - owner = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(owner); - if (err) { - goto pop_1_error; - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_DEREF); - PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - // Can't use ERROR_IF here. - // Fortunately we don't need its superpower. - PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); - if (oldobj == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(oldobj); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_FAST); - _PyStackRef v = GETLOCAL(oparg); - if (PyStackRef_IsNull(v)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - SETLOCAL(oparg, PyStackRef_NULL); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_GLOBAL); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Pop(GLOBALS(), name, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - // Can't use ERROR_IF here. - if (err < 0) { - goto error; - } - if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_NAME); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals when deleting %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_DelItem(ns, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - // Can't use ERROR_IF here. - if (err != 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_SUBSCR); - _PyStackRef container; - _PyStackRef sub; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - /* del container[sub] */ - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), - PyStackRef_AsPyObjectBorrow(sub)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (err) { - goto pop_2_error; - } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_MERGE); - _PyStackRef callable; - _PyStackRef dict; - _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - callable = stack_pointer[-5 - (oparg - 1)]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_MergeEx(dict_o, update_o, 2); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatKwargsError(tstate, callable_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(update); - goto pop_1_error; - } - PyStackRef_CLOSE(update); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_UPDATE); - _PyStackRef dict; - _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Update(dict_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update_o)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - PyStackRef_CLOSE(update); - goto pop_1_error; - } - PyStackRef_CLOSE(update); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(END_ASYNC_FOR); - _PyStackRef awaitable_st; - _PyStackRef exc_st; - exc_st = stack_pointer[-1]; - awaitable_st = stack_pointer[-2]; - PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); - assert(exc && PyExceptionInstance_Check(exc)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - PyStackRef_CLOSE(awaitable_st); - PyStackRef_CLOSE(exc_st); - } - else { - Py_INCREF(exc); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto exception_unwind; - } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - next_instr += 1; - INSTRUCTION_STATS(END_FOR); - _PyStackRef value; - value = stack_pointer[-1]; - /* Don't update instr_ptr, so that POP_ITER sees - * the FOR_ITER as the previous instruction. - * This has the benign side effect that if value is - * finalized it will see the location as the FOR_ITER's. - */ - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_SEND); - _PyStackRef receiver; - _PyStackRef value; - _PyStackRef val; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - (void)receiver; - val = value; - PyStackRef_CLOSE(receiver); - stack_pointer[-2] = val; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(ENTER_EXECUTOR); - opcode = ENTER_EXECUTOR; - #ifdef _Py_TIER2 - PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; - assert(executor->vm_data.index == INSTR_OFFSET() - 1); - assert(executor->vm_data.code == code); - assert(executor->vm_data.valid); - assert(tstate->previous_executor == NULL); - /* If the eval breaker is set then stay in tier 1. - * This avoids any potentially infinite loops - * involving _RESUME_CHECK */ - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - opcode = executor->vm_data.opcode; - oparg = (oparg & ~255) | executor->vm_data.oparg; - next_instr = this_instr; - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - } - DISPATCH_GOTO(); - } - tstate->previous_executor = Py_None; - Py_INCREF(executor); - GOTO_TIER_TWO(executor); - #else - Py_FatalError("ENTER_EXECUTOR is not supported in this build"); - #endif /* _Py_TIER2 */ - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXIT_INIT_CHECK); - _PyStackRef should_be_none; - should_be_none = stack_pointer[-1]; - assert(STACK_LEVEL() == 2); - if (!PyStackRef_IsNone(should_be_none)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyErr_Format(PyExc_TypeError, - "__init__() should return None, not '%.200s'", - Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXTENDED_ARG); - opcode = EXTENDED_ARG; - assert(oparg); - opcode = next_instr->op.code; - oparg = oparg << 8 | next_instr->op.arg; - PRE_DISPATCH_GOTO(); - DISPATCH_GOTO(); - } - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_SIMPLE); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - /* If value is a unicode object, then we know the result - * of format(value) is value itself. */ - if (!PyUnicode_CheckExact(value_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Format(value_o, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - else { - res = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_WITH_SPEC); - _PyStackRef value; - _PyStackRef fmt_spec; - _PyStackRef res; - fmt_spec = stack_pointer[-1]; - value = stack_pointer[-2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - PyStackRef_CLOSE(fmt_spec); - if (res_o == NULL) { - goto pop_2_error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER); - PREDICTED_FOR_ITER:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef iter; - _PyStackRef next; - // _SPECIALIZE_FOR_ITER - { - iter = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ForIter(iter, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(FOR_ITER); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ - } - // _FOR_ITER - { - /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_o == NULL) { - if (_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - /* Jump forward oparg, then skip following END_FOR */ - JUMPBY(oparg + 1); - DISPATCH(); - } - next = PyStackRef_FromPyObjectSteal(next_o); - // Common case: no jump, leave it to the code generator - } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_GEN); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyInterpreterFrame *gen_frame; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _FOR_ITER_GEN_FRAME - { - iter = stack_pointer[-1]; - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(gen) != &PyGen_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, oparg); - } - if (gen->gi_frame_state >= FRAME_EXECUTING) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(FOR_ITER, hit); - gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, PyStackRef_None); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - gen_frame->previous = frame; - // oparg is the return offset from the next instruction. - frame->return_offset = (uint16_t)( 2 + oparg); - } - // _PUSH_FRAME - { - new_frame = gen_frame; - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_LIST); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_LIST - { - iter = stack_pointer[-1]; - if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _ITER_JUMP_LIST - { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - STAT_INC(FOR_ITER, hit); - PyListObject *seq = it->it_seq; - if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { - it->it_index = -1; - #ifndef Py_GIL_DISABLED - if (seq != NULL) { - it->it_seq = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - #endif - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - } - } - // _ITER_NEXT_LIST - { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyList_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); - } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_RANGE); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_RANGE - { - iter = stack_pointer[-1]; - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(r) != &PyRangeIter_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _ITER_JUMP_RANGE - { - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - STAT_INC(FOR_ITER, hit); - if (r->len <= 0) { - // Jump over END_FOR instruction. - JUMPBY(oparg + 1); - DISPATCH(); - } - } - // _ITER_NEXT_RANGE - { - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - assert(r->len > 0); - long value = r->start; - r->start = value + r->step; - r->len--; - PyObject *res = PyLong_FromLong(value); - if (res == NULL) { - goto error; - } - next = PyStackRef_FromPyObjectSteal(res); - } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_TUPLE); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_TUPLE - { - iter = stack_pointer[-1]; - if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - Py_MUSTTAIL return _TAIL_CALL_FOR_ITER(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _ITER_JUMP_TUPLE - { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - STAT_INC(FOR_ITER, hit); - PyTupleObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { - if (seq != NULL) { - it->it_seq = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - } - } - // _ITER_NEXT_TUPLE - { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - PyTupleObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); - } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AITER); - _PyStackRef obj; - _PyStackRef iter; - obj = stack_pointer[-1]; - unaryfunc getter = NULL; - PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); - PyObject *iter_o; - PyTypeObject *type = Py_TYPE(obj_o); - if (type->tp_as_async != NULL) { - getter = type->tp_as_async->am_aiter; - } - if (getter == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an object with " - "__aiter__ method, got %.100s", - type->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(obj); - goto pop_1_error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - iter_o = (*getter)(obj_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(obj); - if (iter_o == NULL) { - goto pop_1_error; - } - if (Py_TYPE(iter_o)->tp_as_async == NULL || - Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' received an object from __aiter__ " - "that does not implement __anext__: %.100s", - Py_TYPE(iter_o)->tp_name); - Py_DECREF(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ANEXT); - _PyStackRef aiter; - _PyStackRef awaitable; - aiter = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (awaitable_o == NULL) { - goto error; - } - awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); - stack_pointer[0] = awaitable; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AWAITABLE); - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(iterable); - if (iter_o == NULL) { - goto pop_1_error; - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ITER); - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - /* before: [obj]; after [getiter(obj)] */ - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(iterable); - if (iter_o == NULL) { - goto pop_1_error; - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_LEN); - _PyStackRef obj; - _PyStackRef len; - obj = stack_pointer[-1]; - // PUSH(len(TOS)) - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) { - goto error; - } - PyObject *len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) { - goto error; - } - len = PyStackRef_FromPyObjectSteal(len_o); - stack_pointer[0] = len; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_YIELD_FROM_ITER); - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - /* before: [obj]; after [getiter(obj)] */ - PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); - if (PyCoro_CheckExact(iterable_o)) { - /* `iterable` is a coroutine */ - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { - /* and it is used in a 'yield from' expression of a - regular generator. */ - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_TypeError, - "cannot 'yield from' a coroutine object " - "in a non-coroutine generator"); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - iter = iterable; - } - else { - if (PyGen_CheckExact(iterable_o)) { - iter = iterable; - } - else { - /* `iterable` is not a generator. */ - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(iterable_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - goto error; - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - PyStackRef_CLOSE(iterable); - } - } - stack_pointer[-1] = iter; - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_FROM); - _PyStackRef from; - _PyStackRef res; - from = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_NAME); - _PyStackRef level; - _PyStackRef fromlist; - _PyStackRef res; - fromlist = stack_pointer[-1]; - level = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportName(tstate, frame, name, - PyStackRef_AsPyObjectBorrow(fromlist), - PyStackRef_AsPyObjectBorrow(level)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(level); - PyStackRef_CLOSE(fromlist); - if (res_o == NULL) { - goto pop_2_error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL); - opcode = INSTRUMENTED_CALL; - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef *func; - _PyStackRef *maybe_self; - _PyStackRef res; - /* Skip 3 cache entries */ - // _MAYBE_EXPAND_METHOD - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - func = &stack_pointer[-2 - oparg]; - maybe_self = &stack_pointer[-1 - oparg]; - if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _MONITOR_CALL - { - args = &stack_pointer[-oparg]; - maybe_self = &stack_pointer[-1 - oparg]; - func = &stack_pointer[-2 - oparg]; - int is_meth = !PyStackRef_IsNull(maybe_self[0]); - PyObject *function = PyStackRef_AsPyObjectBorrow(func[0]); - PyObject *arg0; - if (is_meth) { - arg0 = PyStackRef_AsPyObjectBorrow(maybe_self[0]); - } - else { - if (oparg) { - arg0 = PyStackRef_AsPyObjectBorrow(args[0]); - } - else { - arg0 = &_PyInstrumentation_MISSING; - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg0 - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; - } - } - // _DO_CALL - { - self_or_null = maybe_self; - callable = func; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - // Check if the call can be inlined or not - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - arguments, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - goto error; - } - frame->return_offset = 4 ; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(res_o); - } - } - } - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - goto error; - } - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - opcode = INSTRUMENTED_CALL_FUNCTION_EX; - _PyStackRef func; - _PyStackRef callargs; - _PyStackRef kwargs_in; - _PyStackRef tuple; - _PyStackRef kwargs_out; - _PyStackRef func_st; - _PyStackRef null; - _PyStackRef callargs_st; - _PyStackRef kwargs_st; - _PyStackRef result; - // _MAKE_CALLARGS_A_TUPLE - { - kwargs_in = stack_pointer[-1]; - callargs = stack_pointer[-2]; - func = stack_pointer[-4]; - PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); - if (PyTuple_CheckExact(callargs_o)) { - tuple = callargs; - kwargs_out = kwargs_in; - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (tuple_o == NULL) { - goto error; - } - kwargs_out = kwargs_in; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - tuple = PyStackRef_FromPyObjectSteal(tuple_o); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - } - } - // _DO_CALL_FUNCTION_EX - { - kwargs_st = kwargs_out; - callargs_st = tuple; - null = stack_pointer[-3]; - func_st = func; - (void)null; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - PyObject *result_o; - assert(!_PyErr_Occurred(tstate)); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - assert(PyTuple_CheckExact(callargs)); - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - stack_pointer[-2] = callargs_st; - stack_pointer[-1] = kwargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(result_o); - } - } - } - } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( - tstate, func_st, locals, - nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - // Need to sync the stack since we exit with DISPATCH_INLINED. - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { - goto error; - } - assert( 1 == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); - } - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[-2] = callargs_st; - stack_pointer[-1] = kwargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result_o == NULL) { - goto error; - } - result = PyStackRef_FromPyObjectSteal(result_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - goto error; - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - } - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); - opcode = INSTRUMENTED_CALL_KW; - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef kwnames_in; - _PyStackRef *func; - _PyStackRef *maybe_self; - _PyStackRef kwnames_out; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _MONITOR_CALL_KW - { - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - int is_meth = !PyStackRef_IsNull(self_or_null[0]); - PyObject *arg; - if (is_meth) { - arg = PyStackRef_AsPyObjectBorrow(self_or_null[0]); - } - else { - if (args) { - arg = PyStackRef_AsPyObjectBorrow(args[0]); - } - else { - arg = &_PyInstrumentation_MISSING; - } - } - PyObject *function = PyStackRef_AsPyObjectBorrow(callable[0]); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; - } - } - // _MAYBE_EXPAND_METHOD_KW - { - kwnames_in = stack_pointer[-1]; - func = &stack_pointer[-3 - oparg]; - maybe_self = &stack_pointer[-2 - oparg]; - if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - kwnames_out = kwnames_in; - } - // _DO_CALL_KW - { - kwnames = kwnames_out; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - // Check if the call can be inlined or not - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-1] = kwnames; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - // Sync stack explicitly since we leave using DISPATCH_INLINED(). - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - goto error; - } - assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); - frame->return_offset = 4 ; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - PyStackRef_CLOSE(kwnames); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - stack_pointer[-1] = kwnames; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(res_o); - } - } - } - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - PyStackRef_CLOSE(kwnames); - if (res_o == NULL) { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_FOR); - _PyStackRef receiver; - _PyStackRef value; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - /* Need to create a fake StopIteration error here, - * to conform to PEP 380 */ - if (PyStackRef_GenCheck(receiver)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; - } - } - PyStackRef_CLOSE(value); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_SEND); - _PyStackRef receiver; - _PyStackRef value; - _PyStackRef val; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; - } - } - val = value; - stack_pointer[-2] = val; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(receiver); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); - /* Skip 1 cache entry */ - _PyStackRef iter_stackref = TOP(); - PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next != NULL) { - PUSH(PyStackRef_FromPyObjectSteal(next)); - INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); - } - else { - if (_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - /* Skip END_FOR */ - JUMPBY(oparg + 1); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); - opcode = INSTRUMENTED_INSTRUCTION; - _PyFrame_SetStackPointer(frame, stack_pointer); - int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_opcode < 0) { - goto error; - } - next_instr = this_instr; - if (_PyOpcode_Caches[next_opcode]) { - PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); - } - assert(next_opcode > 0 && next_opcode < 256); - opcode = next_opcode; - DISPATCH_GOTO(); - } - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); - /* Skip 1 cache entry */ - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - goto error; - } - } - } - // _MONITOR_JUMP_BACKWARD - { - INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_LINE); - opcode = INSTRUMENTED_LINE; - int original_opcode = 0; - if (tstate->tracing) { - PyCodeObject *code = _PyFrame_GetCode(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - int index = (int)(this_instr - _PyFrame_GetBytecode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); - original_opcode = code->_co_monitoring->lines->data[index*code->_co_monitoring->lines->bytes_per_entry]; - next_instr = this_instr; - } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - original_opcode = _Py_call_instrumentation_line( - tstate, frame, this_instr, prev_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (original_opcode < 0) { - next_instr = this_instr+1; - goto error; - } - next_instr = frame->instr_ptr; - if (next_instr != this_instr) { - DISPATCH(); - } - } - if (_PyOpcode_Caches[original_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - /* Prevent the underlying instruction from specializing - * and overwriting the instrumentation. */ - PAUSE_ADAPTIVE_COUNTER(cache->counter); - } - opcode = original_opcode; - DISPATCH_GOTO(); - } - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); - opcode = INSTRUMENTED_LOAD_SUPER_ATTR; - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _LOAD_SUPER_ATTR - { - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - goto pop_3_error; - } - } - // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = {class, self}; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(super); - } - } - } - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - if (super == NULL) { - goto pop_3_error; - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(super, name); - Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - goto error; - } - attr = PyStackRef_FromPyObjectSteal(attr_o); - } - // _PUSH_NULL_CONDITIONAL - { - null = PyStackRef_NULL; - } - stack_pointer[0] = attr; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); - (void)this_instr; // INSTRUMENTED_JUMP requires this_instr - INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); - _PyStackRef iter; - iter = stack_pointer[-1]; - INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); - /* Skip 1 cache entry */ - _PyStackRef cond = POP(); - assert(PyStackRef_BoolCheck(cond)); - int jump = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); - /* Skip 1 cache entry */ - _PyStackRef value_stackref = POP(); - int jump = PyStackRef_IsNone(value_stackref); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value_stackref); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); - /* Skip 1 cache entry */ - _PyStackRef value_stackref = POP(); - int jump = !PyStackRef_IsNone(value_stackref); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value_stackref); - stack_pointer = _PyFrame_GetStackPointer(frame); - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); - /* Skip 1 cache entry */ - _PyStackRef cond = POP(); - assert(PyStackRef_BoolCheck(cond)); - int jump = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RESUME); - // _LOAD_BYTECODE - { - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) { - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; - frame->instr_ptr = bytecode + off; - // Make sure this_instr gets reset correctley for any uops that - // follow - next_instr = frame->instr_ptr; - DISPATCH(); - } - #endif - } - // _MAYBE_INSTRUMENT - { - if (tstate->tracing == 0) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; - } - next_instr = this_instr; - DISPATCH(); - } - } - } - // _CHECK_PERIODIC_IF_NOT_YIELD_FROM - { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - goto error; - } - } - } - } - // _MONITOR_RESUME - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; - } - if (frame->instr_ptr != this_instr) { - /* Instrumentation has jumped */ - next_instr = frame->instr_ptr; - } - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); - _PyStackRef val; - _PyStackRef retval; - _PyStackRef res; - // _RETURN_VALUE_EVENT - { - val = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; - } - } - // _RETURN_VALUE - { - retval = val; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(EMPTY()); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(frame->return_offset); - res = temp; - LLTRACE_RESUME_FRAME(); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); - _PyStackRef val; - _PyStackRef retval; - _PyStackRef value; - // _YIELD_VALUE_EVENT - { - val = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; - } - if (frame->instr_ptr != this_instr) { - next_instr = frame->instr_ptr; - DISPATCH(); - } - } - // _YIELD_VALUE - { - retval = val; - // NOTE: It's important that YIELD_VALUE never raises an exception! - // The compiler treats any exception raised here as a failed close() - // or throw() call. - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - value = temp; - LLTRACE_RESUME_FRAME(); - } - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INTERPRETER_EXIT); - _PyStackRef retval; - retval = stack_pointer[-1]; - assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); - assert(_PyFrame_IsIncomplete(frame)); - /* Restore previous frame and return. */ - tstate->current_frame = frame->previous; - assert(!_PyErr_Occurred(tstate)); - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - PyObject *result = PyStackRef_AsPyObjectSteal(retval); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - /* Not strictly necessary, but prevents warnings */ - return result; - } - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IS_OP); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD); - PREDICTED_JUMP_BACKWARD:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - /* Skip 1 cache entry */ - // _SPECIALIZE_JUMP_BACKWARD - { - #if ENABLE_SPECIALIZATION - if (this_instr->op.code == JUMP_BACKWARD) { - this_instr->op.code = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; - // Need to re-dispatch so the warmup counter isn't off by one: - next_instr = this_instr; - DISPATCH_SAME_OPARG(); - } - #endif - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - goto error; - } - } - } - // _JUMP_BACKWARD_NO_INTERRUPT - { - /* This bytecode is used in the `yield from` or `await` loop. - * If there is an interrupt, we want it handled in the innermost - * generator or coroutine, so we deliberately do not check it here. - * (see bpo-30039). - */ - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD_JIT); - static_assert(1 == 1, "incorrect cache size"); - /* Skip 1 cache entry */ - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - goto error; - } - } - } - // _JUMP_BACKWARD_NO_INTERRUPT - { - /* This bytecode is used in the `yield from` or `await` loop. - * If there is an interrupt, we want it handled in the innermost - * generator or coroutine, so we deliberately do not check it here. - * (see bpo-30039). - */ - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); - } - // _JIT - { - #ifdef _Py_TIER2 - _Py_BackoffCounter counter = this_instr[1].counter; - if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { - _Py_CODEUNIT *start = this_instr; - /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ - while (oparg > 255) { - oparg >>= 8; - start--; - } - _PyExecutorObject *executor; - _PyFrame_SetStackPointer(frame, stack_pointer); - int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (optimized <= 0) { - this_instr[1].counter = restart_backoff_counter(counter); - if (optimized < 0) { - goto error; - } - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - this_instr[1].counter = initial_jump_backoff_counter(); - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(tstate->previous_executor == NULL); - tstate->previous_executor = Py_None; - GOTO_TIER_TWO(executor); - } - } - else { - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - } - #endif - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); - /* This bytecode is used in the `yield from` or `await` loop. - * If there is an interrupt, we want it handled in the innermost - * generator or coroutine, so we deliberately do not check it here. - * (see bpo-30039). - */ - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); - static_assert(1 == 1, "incorrect cache size"); - /* Skip 1 cache entry */ - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - goto error; - } - } - } - // _JUMP_BACKWARD_NO_INTERRUPT - { - /* This bytecode is used in the `yield from` or `await` loop. - * If there is an interrupt, we want it handled in the innermost - * generator or coroutine, so we deliberately do not check it here. - * (see bpo-30039). - */ - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_FORWARD); - JUMPBY(oparg); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_APPEND); - _PyStackRef list; - _PyStackRef v; - v = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), - PyStackRef_AsPyObjectSteal(v)); - if (err < 0) { - goto pop_1_error; - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_EXTEND); - _PyStackRef list_st; - _PyStackRef iterable_st; - iterable_st = stack_pointer[-1]; - list_st = stack_pointer[-2 - (oparg-1)]; - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (none_val == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) - { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Clear(tstate); - _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - PyStackRef_CLOSE(iterable_st); - goto pop_1_error; - } - assert(Py_IsNone(none_val)); - PyStackRef_CLOSE(iterable_st); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR); - PREDICTED_LOAD_ATTR:; - _Py_CODEUNIT* const this_instr = next_instr - 10; - (void)this_instr; - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *self_or_null; - // _SPECIALIZE_LOAD_ATTR - { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 8 cache entries */ - // _LOAD_ATTR - { - self_or_null = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - PyObject *attr_o; - if (oparg & 1) { - /* Designed to work in tandem with CALL, pushes two values. */ - attr_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (is_meth) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - meth | self | arg1 | ... | argN - */ - assert(attr_o != NULL); // No errors on this branch - self_or_null[0] = owner; // Transfer ownership - } - else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - meth | NULL | arg1 | ... | argN - */ - PyStackRef_CLOSE(owner); - if (attr_o == NULL) { - goto pop_1_error; - } - self_or_null[0] = PyStackRef_NULL; - } - } - else { - /* Classic, pushes one value. */ - _PyFrame_SetStackPointer(frame, stack_pointer); - attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(owner); - if (attr_o == NULL) { - goto pop_1_error; - } - } - attr = PyStackRef_FromPyObjectSteal(attr_o); - } - stack_pointer[-1] = attr; - stack_pointer += (oparg&1); - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _CHECK_ATTR_CLASS - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!PyType_Check(owner_o)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_CLASS - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = PyStackRef_FromPyObjectNew(descr); - PyStackRef_CLOSE(owner); - } - // _PUSH_NULL_CONDITIONAL - { - null = PyStackRef_NULL; - } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _CHECK_ATTR_CLASS - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!PyType_Check(owner_o)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _GUARD_TYPE_VERSION - { - uint32_t type_version = read_u32(&this_instr[4].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _LOAD_ATTR_CLASS - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = PyStackRef_FromPyObjectNew(descr); - PyStackRef_CLOSE(owner); - } - // _PUSH_NULL_CONDITIONAL - { - null = PyStackRef_NULL; - } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - /* Skip 1 cache entry */ - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *getattribute = read_obj(&this_instr[6].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert((oparg & 1) == 0); - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - PyTypeObject *cls = Py_TYPE(owner_o); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)getattribute; - assert(func_version != 0); - if (f->func_version != func_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 2); - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( - tstate, PyStackRef_FromPyObjectNew(f), 2, frame); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); - new_frame->localsplus[0] = owner; - new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); - frame->return_offset = 10 ; - DISPATCH_INLINED(new_frame); - } - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _CHECK_MANAGED_OBJECT_HAS_VALUES - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _LOAD_ATTR_INSTANCE_VALUE - { - uint16_t offset = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); - if (attr_o == NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - #ifdef Py_GIL_DISABLED - if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - stack_pointer[-1] = attr; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = PyStackRef_NULL; - } - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _CHECK_ATTR_METHOD_LAZY_DICT - { - uint16_t dictoffset = read_u16(&this_instr[4].cache); - char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); - /* This object has a __dict__, just not yet created */ - if (dict != NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - /* Skip 1 cache entry */ - // _LOAD_ATTR_METHOD_LAZY_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_METHOD_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyDictValues *ivs = _PyObject_InlineValues(owner_o); - if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _LOAD_ATTR_METHOD_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - /* Cached method object */ - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_MODULE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - PyDictKeysObject *mod_keys; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _CHECK_ATTR_MODULE_PUSH_KEYS - { - owner = stack_pointer[-1]; - uint32_t dict_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; - assert(dict != NULL); - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - mod_keys = keys; - } - // _LOAD_ATTR_MODULE_FROM_KEYS - { - uint16_t index = read_u16(&this_instr[4].cache); - assert(mod_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries)); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); - // Clear mod_keys from stack in case we need to deopt - if (attr_o == NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); - if (!increfed) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - #else - Py_INCREF(attr_o); - attr = PyStackRef_FromPyObjectSteal(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - stack_pointer[-1] = attr; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = PyStackRef_NULL; - } - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - PyStackRef_CLOSE(owner); - attr = PyStackRef_FromPyObjectNew(descr); - } - stack_pointer[-1] = attr; - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyDictValues *ivs = _PyObject_InlineValues(owner_o); - if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - PyStackRef_CLOSE(owner); - attr = PyStackRef_FromPyObjectNew(descr); - } - stack_pointer[-1] = attr; - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_PROPERTY_FRAME - { - PyObject *fget = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - assert(Py_IS_TYPE(fget, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)fget; - PyCodeObject *code = (PyCodeObject *)f->func_code; - if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - if (code->co_kwonlyargcount) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - if (code->co_argcount != 1) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(LOAD_ATTR, hit); - new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); - new_frame->localsplus[0] = owner; - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_SLOT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _LOAD_ATTR_SLOT - { - uint16_t index = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **addr = (PyObject **)((char *)owner_o + index); - PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); - if (attr_o == NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); - if (!increfed) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - PyStackRef_CLOSE(owner); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = PyStackRef_NULL; - } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - PyDictObject *dict; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _CHECK_ATTR_WITH_HINT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); - if (dict_o == NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - assert(PyDict_CheckExact((PyObject *)dict_o)); - dict = dict_o; - } - // _LOAD_ATTR_WITH_HINT - { - uint16_t hint = read_u16(&this_instr[4].cache); - PyObject *attr_o; - if (!LOCK_OBJECT(dict)) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - if (hint >= (size_t)dict->ma_keys->dk_nentries) { - UNLOCK_OBJECT(dict); - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { - UNLOCK_OBJECT(dict); - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) { - UNLOCK_OBJECT(dict); - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - attr_o = ep->me_value; - if (attr_o == NULL) { - UNLOCK_OBJECT(dict); - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - STAT_INC(LOAD_ATTR, hit); - attr = PyStackRef_FromPyObjectNew(attr_o); - UNLOCK_OBJECT(dict); - PyStackRef_CLOSE(owner); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = PyStackRef_NULL; - } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_BUILD_CLASS); - _PyStackRef bc; - PyObject *bc_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - goto error; - } - if (bc_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_NameError, - "__build_class__ not found"); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - bc = PyStackRef_FromPyObjectSteal(bc_o); - stack_pointer[0] = bc; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); - _PyStackRef value; - // Keep in sync with _common_constants in opcode.py - // If we ever have more than two constants, use a lookup table - PyObject *val; - if (oparg == CONSTANT_ASSERTIONERROR) { - val = PyExc_AssertionError; - } - else { - assert(oparg == CONSTANT_NOTIMPLEMENTEDERROR); - val = PyExc_NotImplementedError; - } - value = PyStackRef_FromPyObjectImmortal(val); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST); - PREDICTED_LOAD_CONST:; - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; - _PyStackRef value; - /* We can't do this in the bytecode compiler as - * marshalling can intern strings and make them immortal. */ - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectNew(obj); - #if ENABLE_SPECIALIZATION_FT - #ifdef Py_GIL_DISABLED - uint8_t expected = LOAD_CONST; - if (!_Py_atomic_compare_exchange_uint8( - &this_instr->op.code, &expected, - _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL)) { - // We might lose a race with instrumentation, which we don't care about. - assert(expected >= MIN_INSTRUMENTED_OPCODE); - } - #else - if (this_instr->op.code == LOAD_CONST) { - this_instr->op.code = _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL; - } - #endif - #endif - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST_IMMORTAL); - static_assert(0 == 0, "incorrect cache size"); - _PyStackRef value; - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - assert(_Py_IsImmortal(obj)); - value = PyStackRef_FromPyObjectImmortal(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_MORTAL(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST_MORTAL); - static_assert(0 == 0, "incorrect cache size"); - _PyStackRef value; - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectNew(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_DEREF); - _PyStackRef value; - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *value_o = PyCell_GetRef(cell); - if (value_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - value = PyStackRef_FromPyObjectSteal(value_o); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST); - _PyStackRef value; - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); - _PyStackRef value; - value = GETLOCAL(oparg); - // do not use SETLOCAL here, it decrefs the old value - GETLOCAL(oparg) = PyStackRef_NULL; - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_CHECK); - _PyStackRef value; - _PyStackRef value_s = GETLOCAL(oparg); - if (PyStackRef_IsNull(value_s)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - value = PyStackRef_DUP(value_s); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); - _PyStackRef value1; - _PyStackRef value2; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - value1 = PyStackRef_DUP(GETLOCAL(oparg1)); - value2 = PyStackRef_DUP(GETLOCAL(oparg2)); - stack_pointer[0] = value1; - stack_pointer[1] = value2; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); - _PyStackRef class_dict_st; - _PyStackRef value; - class_dict_st = stack_pointer[-1]; - PyObject *value_o; - PyObject *name; - PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); - assert(class_dict); - assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); - name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - goto error; - } - if (!value_o) { - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - value_o = PyCell_GetRef(cell); - if (value_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(class_dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - value = PyStackRef_FromPyObjectSteal(value_o); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); - _PyStackRef mod_or_class_dict; - _PyStackRef v; - mod_or_class_dict = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *v_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(mod_or_class_dict); - if (err < 0) { - goto pop_1_error; - } - if (v_o == NULL) { - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - goto error; - } - } - else { - /* Slow-path if globals or builtins is not a dict */ - /* namespace 1: globals */ - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - goto error; - } - if (v_o == NULL) { - /* namespace 2: builtins */ - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - goto error; - } - if (v_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - } - } - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - v = PyStackRef_FromPyObjectSteal(v_o); - stack_pointer[-1] = v; - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL); - PREDICTED_LOAD_GLOBAL:; - _Py_CODEUNIT* const this_instr = next_instr - 5; - (void)this_instr; - _PyStackRef *res; - _PyStackRef null = PyStackRef_NULL; - // _SPECIALIZE_LOAD_GLOBAL - { - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_GLOBAL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 1 cache entry */ - /* Skip 1 cache entry */ - /* Skip 1 cache entry */ - // _LOAD_GLOBAL - { - res = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*res)) { - goto error; - } - } - // _PUSH_NULL_CONDITIONAL - { - null = PyStackRef_NULL; - } - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - PyDictKeysObject *builtins_keys; - _PyStackRef res; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_GLOBALS_VERSION - { - uint16_t version = read_u16(&this_instr[2].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, oparg); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, oparg); - } - assert(DK_IS_UNICODE(keys)); - } - // _GUARD_BUILTINS_VERSION_PUSH_KEYS - { - uint16_t version = read_u16(&this_instr[3].cache); - PyDictObject *dict = (PyDictObject *)BUILTINS(); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, oparg); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, oparg); - } - builtins_keys = keys; - assert(DK_IS_UNICODE(builtins_keys)); - } - // _LOAD_GLOBAL_BUILTINS_FROM_KEYS - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - if (res_o == NULL) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, oparg); - } - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - if (!increfed) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, oparg); - } - #else - Py_INCREF(res_o); - res = PyStackRef_FromPyObjectSteal(res_o); - #endif - STAT_INC(LOAD_GLOBAL, hit); - } - // _PUSH_NULL_CONDITIONAL - { - null = PyStackRef_NULL; - } - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - PyDictKeysObject *globals_keys; - _PyStackRef res; - _PyStackRef null = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_GLOBALS_VERSION_PUSH_KEYS - { - uint16_t version = read_u16(&this_instr[2].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, oparg); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, oparg); - } - globals_keys = keys; - assert(DK_IS_UNICODE(globals_keys)); - } - /* Skip 1 cache entry */ - // _LOAD_GLOBAL_MODULE_FROM_KEYS - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - if (res_o == NULL) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, oparg); - } - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - if (!increfed) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_GLOBAL(frame, stack_pointer, tstate, this_instr, oparg); - } - #else - Py_INCREF(res_o); - res = PyStackRef_FromPyObjectSteal(res_o); - #endif - STAT_INC(LOAD_GLOBAL, hit); - } - // _PUSH_NULL_CONDITIONAL - { - null = PyStackRef_NULL; - } - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_LOCALS); - _PyStackRef locals; - PyObject *l = LOCALS(); - if (l == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - locals = PyStackRef_FromPyObjectNew(l); - stack_pointer[0] = locals; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_NAME); - _PyStackRef v; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *v_o = _PyEval_LoadName(tstate, frame, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) { - goto error; - } - v = PyStackRef_FromPyObjectSteal(v_o); - stack_pointer[0] = v; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_SMALL_INT); - _PyStackRef value; - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectImmortal(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_SPECIAL); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self_or_null; - owner = stack_pointer[-1]; - assert(oparg <= SPECIAL_MAX); - PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); - PyObject *name = _Py_SpecialMethods[oparg].name; - PyObject *self_or_null_o; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = _PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - _Py_SpecialMethods[oparg].error, - Py_TYPE(owner_o)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - goto error; - } - attr = PyStackRef_FromPyObjectSteal(attr_o); - self_or_null = self_or_null_o == NULL ? - PyStackRef_NULL : PyStackRef_FromPyObjectSteal(self_or_null_o); - stack_pointer[0] = attr; - stack_pointer[1] = self_or_null; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR); - PREDICTED_LOAD_SUPER_ATTR:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = LOAD_SUPER_ATTR; - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - // _SPECIALIZE_LOAD_SUPER_ATTR - { - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - int load_method = oparg & 1; - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _LOAD_SUPER_ATTR - { - self_st = stack_pointer[-1]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - goto pop_3_error; - } - } - // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = {class, self}; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(super); - } - } - } - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - if (super == NULL) { - goto pop_3_error; - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(super, name); - Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - goto error; - } - attr = PyStackRef_FromPyObjectSteal(attr_o); - } - // _PUSH_NULL_CONDITIONAL - { - null = PyStackRef_NULL; - } - stack_pointer[0] = attr; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr_st; - /* Skip 1 cache entry */ - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - assert(!(oparg & 1)); - if (global_super != (PyObject *)&PySuper_Type) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_SUPER_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!PyType_Check(class)) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_SUPER_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - if (attr == NULL) { - goto pop_3_error; - } - attr_st = PyStackRef_FromPyObjectSteal(attr); - stack_pointer[-3] = attr_st; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef self_or_null; - /* Skip 1 cache entry */ - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - assert(oparg & 1); - if (global_super != (PyObject *)&PySuper_Type) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_SUPER_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!PyType_Check(class)) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_LOAD_SUPER_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - PyTypeObject *cls = (PyTypeObject *)class; - int method_found = 0; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - goto error; - } - if (method_found) { - self_or_null = self_st; // transfer ownership - } else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - self_or_null = PyStackRef_NULL; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - attr = PyStackRef_FromPyObjectSteal(attr_o); - stack_pointer[-3] = attr; - stack_pointer[-2] = self_or_null; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_CELL); - // "initial" is probably NULL but not if it's an arg (or set - // via the f_locals proxy before MAKE_CELL has run). - PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *cell = PyCell_New(initial); - if (cell == NULL) { - goto error; - } - SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell)); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_FUNCTION); - _PyStackRef codeobj_st; - _PyStackRef func; - codeobj_st = stack_pointer[-1]; - PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(codeobj_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (func_obj == NULL) { - goto error; - } - _PyFunction_SetVersion( - func_obj, ((PyCodeObject *)codeobj)->co_version); - func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); - stack_pointer[0] = func; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAP_ADD); - _PyStackRef dict_st; - _PyStackRef key; - _PyStackRef value; - value = stack_pointer[-1]; - key = stack_pointer[-2]; - dict_st = stack_pointer[-3 - (oparg - 1)]; - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); - /* dict[key] = value */ - // Do not DECREF INPUTS because the function steals the references - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2( - (PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(key), - PyStackRef_AsPyObjectSteal(value) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - goto pop_2_error; - } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_CLASS); - _PyStackRef subject; - _PyStackRef type; - _PyStackRef names; - _PyStackRef attrs; - names = stack_pointer[-1]; - type = stack_pointer[-2]; - subject = stack_pointer[-3]; - // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or - // None on failure. - assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attrs_o = _PyEval_MatchClass(tstate, - PyStackRef_AsPyObjectBorrow(subject), - PyStackRef_AsPyObjectBorrow(type), oparg, - PyStackRef_AsPyObjectBorrow(names)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(subject); - PyStackRef_CLOSE(type); - PyStackRef_CLOSE(names); - if (attrs_o) { - assert(PyTuple_CheckExact(attrs_o)); // Success! - attrs = PyStackRef_FromPyObjectSteal(attrs_o); - } - else { - if (_PyErr_Occurred(tstate)) { - goto pop_3_error; - } - // Error! - attrs = PyStackRef_None; // Failure! - } - stack_pointer[-3] = attrs; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_KEYS); - _PyStackRef subject; - _PyStackRef keys; - _PyStackRef values_or_none; - keys = stack_pointer[-1]; - subject = stack_pointer[-2]; - // On successful match, PUSH(values). Otherwise, PUSH(None). - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, - PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (values_or_none_o == NULL) { - goto error; - } - values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); - stack_pointer[0] = values_or_none; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_MAPPING); - _PyStackRef subject; - _PyStackRef res; - subject = stack_pointer[-1]; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_SEQUENCE); - _PyStackRef subject; - _PyStackRef res; - subject = stack_pointer[-1]; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOP); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOT_TAKEN); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_EXCEPT); - _PyStackRef exc_value; - exc_value = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_XSETREF(exc_info->exc_value, - PyStackRef_IsNone(exc_value) - ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_ITER); - _PyStackRef value; - value = stack_pointer[-1]; - PyStackRef_CLOSE(value); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_FALSE); - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NONE); - _PyStackRef value; - _PyStackRef b; - _PyStackRef cond; - /* Skip 1 cache entry */ - // _IS_NONE - { - value = stack_pointer[-1]; - if (PyStackRef_IsNone(value)) { - b = PyStackRef_True; - } - else { - b = PyStackRef_False; - PyStackRef_CLOSE(value); - } - } - // _POP_JUMP_IF_TRUE - { - cond = b; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); - _PyStackRef value; - _PyStackRef b; - _PyStackRef cond; - /* Skip 1 cache entry */ - // _IS_NONE - { - value = stack_pointer[-1]; - if (PyStackRef_IsNone(value)) { - b = PyStackRef_True; - } - else { - b = PyStackRef_False; - PyStackRef_CLOSE(value); - } - } - // _POP_JUMP_IF_FALSE - { - cond = b; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_TRUE); - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_TOP); - _PyStackRef value; - value = stack_pointer[-1]; - PyStackRef_CLOSE(value); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_EXC_INFO); - _PyStackRef exc; - _PyStackRef prev_exc; - _PyStackRef new_exc; - exc = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - if (exc_info->exc_value != NULL) { - prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); - } - else { - prev_exc = PyStackRef_None; - } - assert(PyStackRef_ExceptionInstanceCheck(exc)); - exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); - new_exc = exc; - stack_pointer[-1] = prev_exc; - stack_pointer[0] = new_exc; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_NULL); - _PyStackRef res; - res = PyStackRef_NULL; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(RAISE_VARARGS); - _PyStackRef *args; - args = &stack_pointer[-oparg]; - assert(oparg < 3); - PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; - PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = do_raise(tstate, exc, cause); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - assert(oparg == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_reraise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto exception_unwind; - } - goto error; - } - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(RERAISE); - _PyStackRef *values; - _PyStackRef exc_st; - exc_st = stack_pointer[-1]; - values = &stack_pointer[-1 - oparg]; - PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); - assert(oparg >= 0 && oparg <= 2); - if (oparg) { - PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]); - if (PyLong_Check(lasti)) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyLong_AsLong(lasti); - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(!_PyErr_Occurred(tstate)); - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); - Py_DECREF(exc); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - assert(exc && PyExceptionInstance_Check(exc)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto exception_unwind; - } - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESERVED); - assert(0 && "Executing RESERVED instruction."); - Py_FatalError("Executing RESERVED instruction."); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME); - PREDICTED_RESUME:; - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; - // _LOAD_BYTECODE - { - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) { - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; - frame->instr_ptr = bytecode + off; - // Make sure this_instr gets reset correctley for any uops that - // follow - next_instr = frame->instr_ptr; - DISPATCH(); - } - #endif - } - // _MAYBE_INSTRUMENT - { - if (tstate->tracing == 0) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; - } - next_instr = this_instr; - DISPATCH(); - } - } - } - // _QUICKEN_RESUME - { - #if ENABLE_SPECIALIZATION_FT - if (tstate->tracing == 0 && this_instr->op.code == RESUME) { - FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); - } - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _CHECK_PERIODIC_IF_NOT_YIELD_FROM - { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - goto error; - } - } - } - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME_CHECK); - static_assert(0 == 0, "incorrect cache size"); - #if defined(__EMSCRIPTEN__) - if (_Py_emscripten_signal_clock == 0) { - UPDATE_MISS_STATS(RESUME); - assert(_PyOpcode_Deopt[opcode] == (RESUME)); - Py_MUSTTAIL return _TAIL_CALL_RESUME(frame, stack_pointer, tstate, this_instr, oparg); - } - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; - #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - if (eval_breaker != version) { - UPDATE_MISS_STATS(RESUME); - assert(_PyOpcode_Deopt[opcode] == (RESUME)); - Py_MUSTTAIL return _TAIL_CALL_RESUME(frame, stack_pointer, tstate, this_instr, oparg); - } - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - UPDATE_MISS_STATS(RESUME); - assert(_PyOpcode_Deopt[opcode] == (RESUME)); - Py_MUSTTAIL return _TAIL_CALL_RESUME(frame, stack_pointer, tstate, this_instr, oparg); - } - #endif - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_GENERATOR); - _PyStackRef res; - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (gen == NULL) { - goto error; - } - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_VALUE); - _PyStackRef retval; - _PyStackRef res; - retval = stack_pointer[-1]; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(EMPTY()); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(frame->return_offset); - res = temp; - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND); - PREDICTED_SEND:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef receiver; - _PyStackRef v; - _PyStackRef retval; - // _SPECIALIZE_SEND - { - receiver = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Send(receiver, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(SEND); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _SEND - { - v = stack_pointer[-1]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - PyObject *retval_o; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - if ((tstate->interp->eval_frame == NULL) && - (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && - ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) - { - PyGenObject *gen = (PyGenObject *)receiver_o; - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert( 2 + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2 + oparg); - assert(gen_frame->previous == NULL); - gen_frame->previous = frame; - DISPATCH_INLINED(gen_frame); - } - if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = PyObject_CallMethodOneArg(receiver_o, - &_Py_ID(send), - PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - if (retval_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyGen_FetchStopIterationValue(&retval_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err == 0) { - assert(retval_o != NULL); - JUMPBY(oparg); - } - else { - PyStackRef_CLOSE(v); - goto pop_1_error; - } - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - retval = PyStackRef_FromPyObjectSteal(retval_o); - } - stack_pointer[0] = retval; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND_GEN); - static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); - _PyStackRef receiver; - _PyStackRef v; - _PyInterpreterFrame *gen_frame; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(SEND); - assert(_PyOpcode_Deopt[opcode] == (SEND)); - Py_MUSTTAIL return _TAIL_CALL_SEND(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _SEND_GEN_FRAME - { - v = stack_pointer[-1]; - receiver = stack_pointer[-2]; - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { - UPDATE_MISS_STATS(SEND); - assert(_PyOpcode_Deopt[opcode] == (SEND)); - Py_MUSTTAIL return _TAIL_CALL_SEND(frame, stack_pointer, tstate, this_instr, oparg); - } - if (gen->gi_frame_state >= FRAME_EXECUTING) { - UPDATE_MISS_STATS(SEND); - assert(_PyOpcode_Deopt[opcode] == (SEND)); - Py_MUSTTAIL return _TAIL_CALL_SEND(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(SEND, hit); - gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert( 2 + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2 + oparg); - gen_frame->previous = frame; - } - // _PUSH_FRAME - { - new_frame = gen_frame; - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SETUP_ANNOTATIONS); - PyObject *ann_dict; - if (LOCALS() == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when setting up annotations"); - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - /* check if __annotations__ in locals()... */ - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - goto error; - } - if (ann_dict == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - ann_dict = PyDict_New(); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (ann_dict == NULL) { - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; - } - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_ADD); - _PyStackRef set; - _PyStackRef v; - v = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), - PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - if (err) { - goto pop_1_error; - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); - _PyStackRef attr_st; - _PyStackRef func_in; - _PyStackRef func_out; - func_in = stack_pointer[-1]; - attr_st = stack_pointer[-2]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); - PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); - func_out = func_in; - assert(PyFunction_Check(func)); - size_t offset = _Py_FunctionAttributeOffsets[oparg]; - assert(offset != 0); - PyObject **ptr = (PyObject **)(((char *)func) + offset); - assert(*ptr == NULL); - *ptr = attr; - stack_pointer[-2] = func_out; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_UPDATE); - _PyStackRef set; - _PyStackRef iterable; - iterable = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), - PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(iterable); - if (err < 0) { - goto pop_1_error; - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR); - PREDICTED_STORE_ATTR:; - _Py_CODEUNIT* const this_instr = next_instr - 5; - (void)this_instr; - _PyStackRef owner; - _PyStackRef v; - // _SPECIALIZE_STORE_ATTR - { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_StoreAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(STORE_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 3 cache entries */ - // _STORE_ATTR - { - v = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), - name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(owner); - if (err) { - goto pop_2_error; - } - } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION_AND_LOCK - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(type_version != 0); - if (!LOCK_OBJECT(owner_o)) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - PyTypeObject *tp = Py_TYPE(owner_o); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UNLOCK_OBJECT(owner_o); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - } - // _GUARD_DORV_NO_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (_PyObject_GetManagedDict(owner_o) || - !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { - UNLOCK_OBJECT(owner_o); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - } - // _STORE_ATTR_INSTANCE_VALUE - { - value = stack_pointer[-2]; - uint16_t offset = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - STAT_INC(STORE_ATTR, hit); - assert(_PyObject_GetManagedDict(owner_o) == NULL); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *old_value = *value_ptr; - FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); - if (old_value == NULL) { - PyDictValues *values = _PyObject_InlineValues(owner_o); - Py_ssize_t index = value_ptr - values->values; - _PyDictValues_AddToInsertionOrder(values, index); - } - UNLOCK_OBJECT(owner_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_SLOT); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _STORE_ATTR_SLOT - { - value = stack_pointer[-2]; - uint16_t index = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!LOCK_OBJECT(owner_o)) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - char *addr = (char *)owner_o + index; - STAT_INC(STORE_ATTR, hit); - PyObject *old_value = *(PyObject **)addr; - FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); - UNLOCK_OBJECT(owner_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _STORE_ATTR_WITH_HINT - { - value = stack_pointer[-2]; - uint16_t hint = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - if (dict == NULL) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!LOCK_OBJECT(dict)) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - #ifdef Py_GIL_DISABLED - if (dict != _PyObject_GetManagedDict(owner_o)) { - UNLOCK_OBJECT(dict); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - #endif - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (hint >= (size_t)dict->ma_keys->dk_nentries || - !DK_IS_UNICODE(dict->ma_keys)) { - UNLOCK_OBJECT(dict); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) { - UNLOCK_OBJECT(dict); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - PyObject *old_value = ep->me_value; - if (old_value == NULL) { - UNLOCK_OBJECT(dict); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_ATTR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); - UNLOCK_OBJECT(dict); - // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, - // when dict only holds the strong reference to value in ep->me_value. - STAT_INC(STORE_ATTR, hit); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_DEREF); - _PyStackRef v; - v = stack_pointer[-1]; - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST); - _PyStackRef value; - value = stack_pointer[-1]; - SETLOCAL(oparg, value); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); - _PyStackRef value1; - _PyStackRef value2; - value1 = stack_pointer[-1]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - SETLOCAL(oparg1, value1); - value2 = PyStackRef_DUP(GETLOCAL(oparg2)); - stack_pointer[-1] = value2; - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_STORE_FAST); - _PyStackRef value2; - _PyStackRef value1; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - SETLOCAL(oparg1, value1); - SETLOCAL(oparg2, value2); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_GLOBAL); - _PyStackRef v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - if (err) { - goto pop_1_error; - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_NAME); - _PyStackRef v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when storing %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - goto pop_1_error; - } - if (PyDict_CheckExact(ns)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - PyStackRef_CLOSE(v); - if (err) { - goto pop_1_error; - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_SLICE); - _PyStackRef v; - _PyStackRef container; - _PyStackRef start; - _PyStackRef stop; - // _SPECIALIZE_STORE_SLICE - { - // Placeholder until we implement STORE_SLICE specialization - #if ENABLE_SPECIALIZATION - OPCODE_DEFERRED_INC(STORE_SLICE); - #endif /* ENABLE_SPECIALIZATION */ - } - // _STORE_SLICE - { - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - v = stack_pointer[-4]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); - stack_pointer = _PyFrame_GetStackPointer(frame); - int err; - if (slice == NULL) { - err = 1; - } - else { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); - Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - } - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(container); - if (err) { - goto pop_4_error; - } - } - stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR); - PREDICTED_STORE_SUBSCR:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef container; - _PyStackRef sub; - _PyStackRef v; - // _SPECIALIZE_STORE_SUBSCR - { - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_StoreSubscr(container, sub, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(STORE_SUBSCR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _STORE_SUBSCR - { - v = stack_pointer[-3]; - /* container[sub] = v */ - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (err) { - goto pop_3_error; - } - } - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_DICT); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef dict_st; - _PyStackRef sub; - /* Skip 1 cache entry */ - sub = stack_pointer[-1]; - dict_st = stack_pointer[-2]; - value = stack_pointer[-3]; - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(STORE_SUBSCR, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(sub), - PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; - } - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef list_st; - _PyStackRef sub_st; - /* Skip 1 cache entry */ - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - value = stack_pointer[-3]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - if (!PyLong_CheckExact(sub)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!PyList_CheckExact(list)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - // Ensure nonnegative, zero-or-one-digit ints. - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (!LOCK_OBJECT(list)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - // Ensure index < len(list) - if (index >= PyList_GET_SIZE(list)) { - UNLOCK_OBJECT(list); - if (true) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - Py_MUSTTAIL return _TAIL_CALL_STORE_SUBSCR(frame, stack_pointer, tstate, this_instr, oparg); - } - } - STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); - PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); - assert(old_value != NULL); - UNLOCK_OBJECT(list); // unlock before decrefs! - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(list_st); - Py_DECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SWAP); - _PyStackRef *bottom; - _PyStackRef *top; - top = &stack_pointer[-1]; - bottom = &stack_pointer[-2 - (oparg-2)]; - _PyStackRef temp = bottom[0]; - bottom[0] = top[0]; - top[0] = temp; - assert(oparg >= 2); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL); - PREDICTED_TO_BOOL:; - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - _PyStackRef value; - _PyStackRef res; - // _SPECIALIZE_TO_BOOL - { - value = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ToBool(value, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(TO_BOOL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _TO_BOOL - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (err < 0) { - goto pop_1_error; - } - res = err ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[-1] = res; - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, oparg); - } - } - // _REPLACE_WITH_TRUE - { - value = owner; - PyStackRef_CLOSE(value); - res = PyStackRef_True; - } - stack_pointer[-1] = res; - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_BOOL); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - if (!PyStackRef_BoolCheck(value)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(TO_BOOL, hit); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_INT); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value_o)) { - assert(_Py_IsImmortal(value_o)); - res = PyStackRef_False; - } - else { - PyStackRef_CLOSE(value); - res = PyStackRef_True; - } - stack_pointer[-1] = res; - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_LIST); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyList_CheckExact(value_o)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(TO_BOOL, hit); - res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; - PyStackRef_CLOSE(value); - stack_pointer[-1] = res; - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_NONE); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - // This one is a bit weird, because we expect *some* failures: - if (!PyStackRef_IsNone(value)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(TO_BOOL, hit); - res = PyStackRef_False; - stack_pointer[-1] = res; - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_STR); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - Py_MUSTTAIL return _TAIL_CALL_TO_BOOL(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(TO_BOOL, hit); - if (value_o == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value_o)); - res = PyStackRef_False; - } - else { - assert(Py_SIZE(value_o)); - PyStackRef_CLOSE(value); - res = PyStackRef_True; - } - stack_pointer[-1] = res; - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_INVERT); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (res_o == NULL) { - goto pop_1_error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NEGATIVE); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (res_o == NULL) { - goto pop_1_error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NOT); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(value)); - res = PyStackRef_IsFalse(value) - ? PyStackRef_True : PyStackRef_False; - stack_pointer[-1] = res; - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNPACK_EX); - _PyStackRef seq; - _PyStackRef *right; - seq = stack_pointer[-1]; - right = &stack_pointer[(oparg & 0xFF)]; - _PyStackRef *top = right + (oparg >> 8); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(seq); - if (res == 0) { - goto pop_1_error; - } - stack_pointer += (oparg & 0xFF) + (oparg >> 8); - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE); - PREDICTED_UNPACK_SEQUENCE:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef seq; - _PyStackRef *output; - // _SPECIALIZE_UNPACK_SEQUENCE - { - seq = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - (void)seq; - (void)counter; - } - // _UNPACK_SEQUENCE - { - output = &stack_pointer[-1]; - _PyStackRef *top = output + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(seq); - if (res == 0) { - goto pop_1_error; - } - } - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef seq; - _PyStackRef *values; - /* Skip 1 cache entry */ - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - if (!PyList_CheckExact(seq_o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, oparg); - } - if (!LOCK_OBJECT(seq_o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, oparg); - } - if (PyList_GET_SIZE(seq_o) != oparg) { - UNLOCK_OBJECT(seq_o); - if (true) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, oparg); - } - } - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectNew(items[i]); - } - UNLOCK_OBJECT(seq_o); - PyStackRef_CLOSE(seq); - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef seq; - _PyStackRef *values; - /* Skip 1 cache entry */ - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - if (!PyTuple_CheckExact(seq_o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, oparg); - } - if (PyTuple_GET_SIZE(seq_o) != oparg) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectNew(items[i]); - } - PyStackRef_CLOSE(seq); - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef seq; - _PyStackRef val1; - _PyStackRef val0; - /* Skip 1 cache entry */ - seq = stack_pointer[-1]; - assert(oparg == 2); - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - if (!PyTuple_CheckExact(seq_o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, oparg); - } - if (PyTuple_GET_SIZE(seq_o) != 2) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - Py_MUSTTAIL return _TAIL_CALL_UNPACK_SEQUENCE(frame, stack_pointer, tstate, this_instr, oparg); - } - STAT_INC(UNPACK_SEQUENCE, hit); - val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); - val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - PyStackRef_CLOSE(seq); - stack_pointer[-1] = val1; - stack_pointer[0] = val0; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(WITH_EXCEPT_START); - _PyStackRef exit_func; - _PyStackRef exit_self; - _PyStackRef lasti; - _PyStackRef val; - _PyStackRef res; - val = stack_pointer[-1]; - lasti = stack_pointer[-3]; - exit_self = stack_pointer[-4]; - exit_func = stack_pointer[-5]; - /* At the top of the stack are 4 values: - - val: TOP = exc_info() - - unused: SECOND = previous exception - - lasti: THIRD = lasti of exception in exc_info() - - exit_self: FOURTH = the context or NULL - - exit_func: FIFTH = the context.__exit__ function or context.__exit__ bound method - We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). - Then we push the __exit__ return value. - */ - PyObject *exc, *tb; - PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); - PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); - assert(val_o && PyExceptionInstance_Check(val_o)); - exc = PyExceptionInstance_Class(val_o); - tb = PyException_GetTraceback(val_o); - if (tb == NULL) { - tb = Py_None; - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(tb); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - assert(PyStackRef_LongCheck(lasti)); - (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; - int has_self = !PyStackRef_IsNull(exit_self); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, - (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - (void)(opcode); - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(YIELD_VALUE); - _PyStackRef retval; - _PyStackRef value; - retval = stack_pointer[-1]; - // NOTE: It's important that YIELD_VALUE never raises an exception! - // The compiler treats any exception raised here as a failed close() - // or throw() call. - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - value = temp; - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - } - DISPATCH(); - pop_4_error: - TAIL_CALL(pop_4_error); - pop_3_error: - TAIL_CALL(pop_3_error); - pop_2_error: - TAIL_CALL(pop_2_error); - pop_1_error: - TAIL_CALL(pop_1_error); - error: - TAIL_CALL(error); - exception_unwind: - TAIL_CALL(exception_unwind); - exit_unwind: - TAIL_CALL(exit_unwind); - start_frame: - TAIL_CALL(start_frame); -} -/* END INSTRUCTIONS */ -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { - { - int opcode = next_instr->op.code; - -_PyErr_Format(tstate, PyExc_SystemError, - "%U:%d: unknown opcode %d", - _PyFrame_GetCode(frame)->co_filename, - PyUnstable_InterpreterFrame_GetLine(frame), - opcode); - } - TAIL_CALL(error); -} -static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { - [BINARY_OP] = _TAIL_CALL_BINARY_OP, - [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, - [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, - [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_BINARY_OP_ADD_UNICODE, - [BINARY_OP_EXTEND] = _TAIL_CALL_BINARY_OP_EXTEND, - [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE, - [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT, - [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_BINARY_OP_MULTIPLY_INT, - [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT, - [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_BINARY_OP_SUBTRACT_INT, - [BINARY_SLICE] = _TAIL_CALL_BINARY_SLICE, - [BINARY_SUBSCR] = _TAIL_CALL_BINARY_SUBSCR, - [BINARY_SUBSCR_DICT] = _TAIL_CALL_BINARY_SUBSCR_DICT, - [BINARY_SUBSCR_GETITEM] = _TAIL_CALL_BINARY_SUBSCR_GETITEM, - [BINARY_SUBSCR_LIST_INT] = _TAIL_CALL_BINARY_SUBSCR_LIST_INT, - [BINARY_SUBSCR_STR_INT] = _TAIL_CALL_BINARY_SUBSCR_STR_INT, - [BINARY_SUBSCR_TUPLE_INT] = _TAIL_CALL_BINARY_SUBSCR_TUPLE_INT, - [BUILD_LIST] = _TAIL_CALL_BUILD_LIST, - [BUILD_MAP] = _TAIL_CALL_BUILD_MAP, - [BUILD_SET] = _TAIL_CALL_BUILD_SET, - [BUILD_SLICE] = _TAIL_CALL_BUILD_SLICE, - [BUILD_STRING] = _TAIL_CALL_BUILD_STRING, - [BUILD_TUPLE] = _TAIL_CALL_BUILD_TUPLE, - [CACHE] = _TAIL_CALL_CACHE, - [CALL] = _TAIL_CALL_CALL, - [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT, - [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS, - [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_CALL_BOUND_METHOD_GENERAL, - [CALL_BUILTIN_CLASS] = _TAIL_CALL_CALL_BUILTIN_CLASS, - [CALL_BUILTIN_FAST] = _TAIL_CALL_CALL_BUILTIN_FAST, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS, - [CALL_BUILTIN_O] = _TAIL_CALL_CALL_BUILTIN_O, - [CALL_FUNCTION_EX] = _TAIL_CALL_CALL_FUNCTION_EX, - [CALL_INTRINSIC_1] = _TAIL_CALL_CALL_INTRINSIC_1, - [CALL_INTRINSIC_2] = _TAIL_CALL_CALL_INTRINSIC_2, - [CALL_ISINSTANCE] = _TAIL_CALL_CALL_ISINSTANCE, - [CALL_KW] = _TAIL_CALL_CALL_KW, - [CALL_KW_BOUND_METHOD] = _TAIL_CALL_CALL_KW_BOUND_METHOD, - [CALL_KW_NON_PY] = _TAIL_CALL_CALL_KW_NON_PY, - [CALL_KW_PY] = _TAIL_CALL_CALL_KW_PY, - [CALL_LEN] = _TAIL_CALL_CALL_LEN, - [CALL_LIST_APPEND] = _TAIL_CALL_CALL_LIST_APPEND, - [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, - [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS, - [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O, - [CALL_NON_PY_GENERAL] = _TAIL_CALL_CALL_NON_PY_GENERAL, - [CALL_PY_EXACT_ARGS] = _TAIL_CALL_CALL_PY_EXACT_ARGS, - [CALL_PY_GENERAL] = _TAIL_CALL_CALL_PY_GENERAL, - [CALL_STR_1] = _TAIL_CALL_CALL_STR_1, - [CALL_TUPLE_1] = _TAIL_CALL_CALL_TUPLE_1, - [CALL_TYPE_1] = _TAIL_CALL_CALL_TYPE_1, - [CHECK_EG_MATCH] = _TAIL_CALL_CHECK_EG_MATCH, - [CHECK_EXC_MATCH] = _TAIL_CALL_CHECK_EXC_MATCH, - [CLEANUP_THROW] = _TAIL_CALL_CLEANUP_THROW, - [COMPARE_OP] = _TAIL_CALL_COMPARE_OP, - [COMPARE_OP_FLOAT] = _TAIL_CALL_COMPARE_OP_FLOAT, - [COMPARE_OP_INT] = _TAIL_CALL_COMPARE_OP_INT, - [COMPARE_OP_STR] = _TAIL_CALL_COMPARE_OP_STR, - [CONTAINS_OP] = _TAIL_CALL_CONTAINS_OP, - [CONTAINS_OP_DICT] = _TAIL_CALL_CONTAINS_OP_DICT, - [CONTAINS_OP_SET] = _TAIL_CALL_CONTAINS_OP_SET, - [CONVERT_VALUE] = _TAIL_CALL_CONVERT_VALUE, - [COPY] = _TAIL_CALL_COPY, - [COPY_FREE_VARS] = _TAIL_CALL_COPY_FREE_VARS, - [DELETE_ATTR] = _TAIL_CALL_DELETE_ATTR, - [DELETE_DEREF] = _TAIL_CALL_DELETE_DEREF, - [DELETE_FAST] = _TAIL_CALL_DELETE_FAST, - [DELETE_GLOBAL] = _TAIL_CALL_DELETE_GLOBAL, - [DELETE_NAME] = _TAIL_CALL_DELETE_NAME, - [DELETE_SUBSCR] = _TAIL_CALL_DELETE_SUBSCR, - [DICT_MERGE] = _TAIL_CALL_DICT_MERGE, - [DICT_UPDATE] = _TAIL_CALL_DICT_UPDATE, - [END_ASYNC_FOR] = _TAIL_CALL_END_ASYNC_FOR, - [END_FOR] = _TAIL_CALL_END_FOR, - [END_SEND] = _TAIL_CALL_END_SEND, - [ENTER_EXECUTOR] = _TAIL_CALL_ENTER_EXECUTOR, - [EXIT_INIT_CHECK] = _TAIL_CALL_EXIT_INIT_CHECK, - [EXTENDED_ARG] = _TAIL_CALL_EXTENDED_ARG, - [FORMAT_SIMPLE] = _TAIL_CALL_FORMAT_SIMPLE, - [FORMAT_WITH_SPEC] = _TAIL_CALL_FORMAT_WITH_SPEC, - [FOR_ITER] = _TAIL_CALL_FOR_ITER, - [FOR_ITER_GEN] = _TAIL_CALL_FOR_ITER_GEN, - [FOR_ITER_LIST] = _TAIL_CALL_FOR_ITER_LIST, - [FOR_ITER_RANGE] = _TAIL_CALL_FOR_ITER_RANGE, - [FOR_ITER_TUPLE] = _TAIL_CALL_FOR_ITER_TUPLE, - [GET_AITER] = _TAIL_CALL_GET_AITER, - [GET_ANEXT] = _TAIL_CALL_GET_ANEXT, - [GET_AWAITABLE] = _TAIL_CALL_GET_AWAITABLE, - [GET_ITER] = _TAIL_CALL_GET_ITER, - [GET_LEN] = _TAIL_CALL_GET_LEN, - [GET_YIELD_FROM_ITER] = _TAIL_CALL_GET_YIELD_FROM_ITER, - [IMPORT_FROM] = _TAIL_CALL_IMPORT_FROM, - [IMPORT_NAME] = _TAIL_CALL_IMPORT_NAME, - [INSTRUMENTED_CALL] = _TAIL_CALL_INSTRUMENTED_CALL, - [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX, - [INSTRUMENTED_CALL_KW] = _TAIL_CALL_INSTRUMENTED_CALL_KW, - [INSTRUMENTED_END_FOR] = _TAIL_CALL_INSTRUMENTED_END_FOR, - [INSTRUMENTED_END_SEND] = _TAIL_CALL_INSTRUMENTED_END_SEND, - [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_INSTRUMENTED_FOR_ITER, - [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_INSTRUMENTED_INSTRUCTION, - [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD, - [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD, - [INSTRUMENTED_LINE] = _TAIL_CALL_INSTRUMENTED_LINE, - [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR, - [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_INSTRUMENTED_NOT_TAKEN, - [INSTRUMENTED_POP_ITER] = _TAIL_CALL_INSTRUMENTED_POP_ITER, - [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE, - [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE, - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, - [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE, - [INSTRUMENTED_RESUME] = _TAIL_CALL_INSTRUMENTED_RESUME, - [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_INSTRUMENTED_RETURN_VALUE, - [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_INSTRUMENTED_YIELD_VALUE, - [INTERPRETER_EXIT] = _TAIL_CALL_INTERPRETER_EXIT, - [IS_OP] = _TAIL_CALL_IS_OP, - [JUMP_BACKWARD] = _TAIL_CALL_JUMP_BACKWARD, - [JUMP_BACKWARD_JIT] = _TAIL_CALL_JUMP_BACKWARD_JIT, - [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT, - [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_JUMP_BACKWARD_NO_JIT, - [JUMP_FORWARD] = _TAIL_CALL_JUMP_FORWARD, - [LIST_APPEND] = _TAIL_CALL_LIST_APPEND, - [LIST_EXTEND] = _TAIL_CALL_LIST_EXTEND, - [LOAD_ATTR] = _TAIL_CALL_LOAD_ATTR, - [LOAD_ATTR_CLASS] = _TAIL_CALL_LOAD_ATTR_CLASS, - [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, - [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE, - [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT, - [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT, - [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES, - [LOAD_ATTR_MODULE] = _TAIL_CALL_LOAD_ATTR_MODULE, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, - [LOAD_ATTR_PROPERTY] = _TAIL_CALL_LOAD_ATTR_PROPERTY, - [LOAD_ATTR_SLOT] = _TAIL_CALL_LOAD_ATTR_SLOT, - [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_LOAD_ATTR_WITH_HINT, - [LOAD_BUILD_CLASS] = _TAIL_CALL_LOAD_BUILD_CLASS, - [LOAD_COMMON_CONSTANT] = _TAIL_CALL_LOAD_COMMON_CONSTANT, - [LOAD_CONST] = _TAIL_CALL_LOAD_CONST, - [LOAD_CONST_IMMORTAL] = _TAIL_CALL_LOAD_CONST_IMMORTAL, - [LOAD_CONST_MORTAL] = _TAIL_CALL_LOAD_CONST_MORTAL, - [LOAD_DEREF] = _TAIL_CALL_LOAD_DEREF, - [LOAD_FAST] = _TAIL_CALL_LOAD_FAST, - [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_LOAD_FAST_AND_CLEAR, - [LOAD_FAST_CHECK] = _TAIL_CALL_LOAD_FAST_CHECK, - [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_LOAD_FAST_LOAD_FAST, - [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF, - [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS, - [LOAD_GLOBAL] = _TAIL_CALL_LOAD_GLOBAL, - [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_LOAD_GLOBAL_BUILTIN, - [LOAD_GLOBAL_MODULE] = _TAIL_CALL_LOAD_GLOBAL_MODULE, - [LOAD_LOCALS] = _TAIL_CALL_LOAD_LOCALS, - [LOAD_NAME] = _TAIL_CALL_LOAD_NAME, - [LOAD_SMALL_INT] = _TAIL_CALL_LOAD_SMALL_INT, - [LOAD_SPECIAL] = _TAIL_CALL_LOAD_SPECIAL, - [LOAD_SUPER_ATTR] = _TAIL_CALL_LOAD_SUPER_ATTR, - [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_LOAD_SUPER_ATTR_ATTR, - [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_LOAD_SUPER_ATTR_METHOD, - [MAKE_CELL] = _TAIL_CALL_MAKE_CELL, - [MAKE_FUNCTION] = _TAIL_CALL_MAKE_FUNCTION, - [MAP_ADD] = _TAIL_CALL_MAP_ADD, - [MATCH_CLASS] = _TAIL_CALL_MATCH_CLASS, - [MATCH_KEYS] = _TAIL_CALL_MATCH_KEYS, - [MATCH_MAPPING] = _TAIL_CALL_MATCH_MAPPING, - [MATCH_SEQUENCE] = _TAIL_CALL_MATCH_SEQUENCE, - [NOP] = _TAIL_CALL_NOP, - [NOT_TAKEN] = _TAIL_CALL_NOT_TAKEN, - [POP_EXCEPT] = _TAIL_CALL_POP_EXCEPT, - [POP_ITER] = _TAIL_CALL_POP_ITER, - [POP_JUMP_IF_FALSE] = _TAIL_CALL_POP_JUMP_IF_FALSE, - [POP_JUMP_IF_NONE] = _TAIL_CALL_POP_JUMP_IF_NONE, - [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_POP_JUMP_IF_NOT_NONE, - [POP_JUMP_IF_TRUE] = _TAIL_CALL_POP_JUMP_IF_TRUE, - [POP_TOP] = _TAIL_CALL_POP_TOP, - [PUSH_EXC_INFO] = _TAIL_CALL_PUSH_EXC_INFO, - [PUSH_NULL] = _TAIL_CALL_PUSH_NULL, - [RAISE_VARARGS] = _TAIL_CALL_RAISE_VARARGS, - [RERAISE] = _TAIL_CALL_RERAISE, - [RESERVED] = _TAIL_CALL_RESERVED, - [RESUME] = _TAIL_CALL_RESUME, - [RESUME_CHECK] = _TAIL_CALL_RESUME_CHECK, - [RETURN_GENERATOR] = _TAIL_CALL_RETURN_GENERATOR, - [RETURN_VALUE] = _TAIL_CALL_RETURN_VALUE, - [SEND] = _TAIL_CALL_SEND, - [SEND_GEN] = _TAIL_CALL_SEND_GEN, - [SETUP_ANNOTATIONS] = _TAIL_CALL_SETUP_ANNOTATIONS, - [SET_ADD] = _TAIL_CALL_SET_ADD, - [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_SET_FUNCTION_ATTRIBUTE, - [SET_UPDATE] = _TAIL_CALL_SET_UPDATE, - [STORE_ATTR] = _TAIL_CALL_STORE_ATTR, - [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE, - [STORE_ATTR_SLOT] = _TAIL_CALL_STORE_ATTR_SLOT, - [STORE_ATTR_WITH_HINT] = _TAIL_CALL_STORE_ATTR_WITH_HINT, - [STORE_DEREF] = _TAIL_CALL_STORE_DEREF, - [STORE_FAST] = _TAIL_CALL_STORE_FAST, - [STORE_FAST_LOAD_FAST] = _TAIL_CALL_STORE_FAST_LOAD_FAST, - [STORE_FAST_STORE_FAST] = _TAIL_CALL_STORE_FAST_STORE_FAST, - [STORE_GLOBAL] = _TAIL_CALL_STORE_GLOBAL, - [STORE_NAME] = _TAIL_CALL_STORE_NAME, - [STORE_SLICE] = _TAIL_CALL_STORE_SLICE, - [STORE_SUBSCR] = _TAIL_CALL_STORE_SUBSCR, - [STORE_SUBSCR_DICT] = _TAIL_CALL_STORE_SUBSCR_DICT, - [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_STORE_SUBSCR_LIST_INT, - [SWAP] = _TAIL_CALL_SWAP, - [TO_BOOL] = _TAIL_CALL_TO_BOOL, - [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TO_BOOL_ALWAYS_TRUE, - [TO_BOOL_BOOL] = _TAIL_CALL_TO_BOOL_BOOL, - [TO_BOOL_INT] = _TAIL_CALL_TO_BOOL_INT, - [TO_BOOL_LIST] = _TAIL_CALL_TO_BOOL_LIST, - [TO_BOOL_NONE] = _TAIL_CALL_TO_BOOL_NONE, - [TO_BOOL_STR] = _TAIL_CALL_TO_BOOL_STR, - [UNARY_INVERT] = _TAIL_CALL_UNARY_INVERT, - [UNARY_NEGATIVE] = _TAIL_CALL_UNARY_NEGATIVE, - [UNARY_NOT] = _TAIL_CALL_UNARY_NOT, - [UNPACK_EX] = _TAIL_CALL_UNPACK_EX, - [UNPACK_SEQUENCE] = _TAIL_CALL_UNPACK_SEQUENCE, - [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_UNPACK_SEQUENCE_LIST, - [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TUPLE, - [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, - [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, - [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, - [118] = _TAIL_CALL_UNKNOWN_OPCODE, - [119] = _TAIL_CALL_UNKNOWN_OPCODE, - [120] = _TAIL_CALL_UNKNOWN_OPCODE, - [121] = _TAIL_CALL_UNKNOWN_OPCODE, - [122] = _TAIL_CALL_UNKNOWN_OPCODE, - [123] = _TAIL_CALL_UNKNOWN_OPCODE, - [124] = _TAIL_CALL_UNKNOWN_OPCODE, - [125] = _TAIL_CALL_UNKNOWN_OPCODE, - [126] = _TAIL_CALL_UNKNOWN_OPCODE, - [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [128] = _TAIL_CALL_UNKNOWN_OPCODE, - [129] = _TAIL_CALL_UNKNOWN_OPCODE, - [130] = _TAIL_CALL_UNKNOWN_OPCODE, - [131] = _TAIL_CALL_UNKNOWN_OPCODE, - [132] = _TAIL_CALL_UNKNOWN_OPCODE, - [133] = _TAIL_CALL_UNKNOWN_OPCODE, - [134] = _TAIL_CALL_UNKNOWN_OPCODE, - [135] = _TAIL_CALL_UNKNOWN_OPCODE, - [136] = _TAIL_CALL_UNKNOWN_OPCODE, - [137] = _TAIL_CALL_UNKNOWN_OPCODE, - [138] = _TAIL_CALL_UNKNOWN_OPCODE, - [139] = _TAIL_CALL_UNKNOWN_OPCODE, - [140] = _TAIL_CALL_UNKNOWN_OPCODE, - [141] = _TAIL_CALL_UNKNOWN_OPCODE, - [142] = _TAIL_CALL_UNKNOWN_OPCODE, - [143] = _TAIL_CALL_UNKNOWN_OPCODE, - [144] = _TAIL_CALL_UNKNOWN_OPCODE, - [145] = _TAIL_CALL_UNKNOWN_OPCODE, - [146] = _TAIL_CALL_UNKNOWN_OPCODE, - [147] = _TAIL_CALL_UNKNOWN_OPCODE, - [148] = _TAIL_CALL_UNKNOWN_OPCODE, - [232] = _TAIL_CALL_UNKNOWN_OPCODE, - [233] = _TAIL_CALL_UNKNOWN_OPCODE, - [234] = _TAIL_CALL_UNKNOWN_OPCODE, -}; -#undef TIER_ONE diff --git a/Python/generated_tail_call_labels.c.h b/Python/generated_tail_call_labels.c.h deleted file mode 100644 index 4671013ec0f810..00000000000000 --- a/Python/generated_tail_call_labels.c.h +++ /dev/null @@ -1,156 +0,0 @@ -// This file is generated by Tools/cases_generator/tier1_tail_call_generator.py -// from: -// Python/bytecodes.c -// Do not edit! - -#ifndef Py_TAIL_CALL_INTERP - #error "This file is for tail-calling interpreter only." -#endif -#define TIER_ONE 1 - - pop_4_error: - { - STACK_SHRINK(4); - goto error; - } - - pop_3_error: - { - STACK_SHRINK(3); - goto error; - } - - pop_2_error: - { - STACK_SHRINK(2); - goto error; - } - - pop_1_error: - { - STACK_SHRINK(1); - goto error; - } - - error: - { - /* Double-check exception status. */ - #ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } - #else - assert(_PyErr_Occurred(tstate)); - #endif - - /* Log traceback info. */ - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); - goto exception_unwind; - } - - exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ - #ifdef LLTRACE - if (frame->lltrace >= 5) { - lltrace_resume_frame(frame); - } - #endif - return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0); - } - - exit_unwind: - { - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - - start_frame: - { - if (_Py_EnterRecursivePy(tstate)) { - goto exit_unwind; - } - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - #ifdef LLTRACE - { - int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); - frame->lltrace = lltrace; - if (lltrace < 0) { - goto exit_unwind; - } - } - #endif - - #ifdef Py_DEBUG - /* _PyEval_EvalFrameDefault() must not be called with an exception set, - because it can clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!_PyErr_Occurred(tstate)); - #endif - return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0); - } - -#undef TIER_ONE diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 8fa26758f964c7..a08b32fa45db3e 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -80,8 +80,6 @@ def clean_lines(text): Python/deepfreeze/*.c Python/frozen_modules/*.h Python/generated_cases.c.h -Python/generated_tail_call_handlers.c.h -Python/generated_tail_call_labels.c.h Python/executor_cases.c.h Python/optimizer_cases.c.h diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 4017a5ebea47ee..00816414022077 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -662,6 +662,7 @@ def has_error_without_pop(op: parser.InstDef) -> bool: "assert", "backoff_counter_triggers", "initial_temperature_backoff_counter", + "JUMP_TO_LABEL", "maybe_lltrace_resume_frame", "restart_backoff_counter", ) diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 6867b2032bc6e9..48a090d6920f57 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -161,7 +161,7 @@ def deopt_if( family_name = inst.family.name self.emit(f"UPDATE_MISS_STATS({family_name});\n") self.emit(f"assert(_PyOpcode_Deopt[opcode] == ({family_name}));\n") - self.emit(f"goto PREDICTED_{family_name};\n") + self.emit(f"JUMP_TO_PREDICTED({family_name});\n") self.emit("}\n") return not always_true(first_tkn) @@ -169,10 +169,10 @@ def deopt_if( def goto_error(self, offset: int, label: str, storage: Storage) -> str: if offset > 0: - return f"goto pop_{offset}_{label};" + return f"JUMP_TO_LABEL(pop_{offset}_{label});" if offset < 0: storage.copy().flush(self.out) - return f"goto {label};" + return f"JUMP_TO_LABEL({label});" def error_if( self, @@ -588,6 +588,8 @@ def _emit_block( if tkn.text.startswith("DISPATCH"): self._print_storage(storage) reachable = False + if tkn.text.startswith("JUMP_TO_LABEL"): + reachable = False self.out.emit(tkn) elif tkn.kind == "IF": self.out.emit(tkn) diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index ed57d65eca956c..2f5042bec9721e 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -27,7 +27,6 @@ from typing import TextIO, Callable from stack import Local, Stack, StackError, get_stack_effect, Storage - DEFAULT_OUTPUT = ROOT / "Python/generated_cases.c.h" @@ -120,90 +119,62 @@ def write_uop( raise analysis_error(ex.args[0], uop.body[0]) -def uses_this(inst: Instruction) -> bool: +def uses_this(inst: Instruction) -> tuple[bool, bool]: if inst.properties.needs_this: - return True + return True, False for uop in inst.parts: if not isinstance(uop, Uop): continue for cache in uop.caches: if cache.name != "unused": - return True - return False + return True, False + # Can't be merged into the loop above, because + # this must strictly be performed at the end. + for uop in inst.parts: + if not isinstance(uop, Uop): + continue + for tkn in uop.body: + if (tkn.kind == "IDENTIFIER" + and (tkn.text in {"DEOPT_IF", "EXIT_IF"})): + return True, True + return False, False -def write_single_inst( - out: CWriter, - emitter: Emitter, - name: str, - inst: Instruction, - uses_this: Callable[[Instruction], bool], - is_in_tail_call: bool = False -) -> None: - if is_in_tail_call: - out.emit("int opcode = next_instr->op.code;\n") - out.emit("(void)(opcode);\n") - needs_this = uses_this(inst) - unused_guard = "(void)this_instr;\n" - if inst.properties.needs_prev: - out.emit(f"_Py_CODEUNIT* const prev_instr = frame->instr_ptr;\n") - if needs_this and not inst.is_target: - if inst.properties.no_save_ip: - out.emit(f"_Py_CODEUNIT* const this_instr = next_instr;\n") - else: - out.emit(f"_Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;\n") - out.emit(unused_guard) - elif not inst.properties.no_save_ip: - out.emit(f"frame->instr_ptr = next_instr;\n") - out.emit(f"next_instr += {inst.size};\n") - out.emit(f"INSTRUCTION_STATS({name});\n") - if inst.is_target: - out.emit(f"PREDICTED_{name}:;\n") - if needs_this: - out.emit(f"_Py_CODEUNIT* const this_instr = next_instr - {inst.size};\n") - out.emit(unused_guard) - if inst.properties.uses_opcode: - out.emit(f"opcode = {name};\n") - if inst.family is not None: - out.emit( - f"static_assert({inst.family.size} == {inst.size-1}" - ', "incorrect cache size");\n' - ) - declare_variables(inst, out) - offset = 1 # The instruction itself - stack = Stack() - for part in inst.parts: - # Only emit braces if more than one uop - insert_braces = len([p for p in inst.parts if isinstance(p, Uop)]) > 1 - offset, stack = write_uop(part, emitter, offset, stack, inst, insert_braces) - out.start_line() - - stack.flush(out) +UNKNOWN_OPCODE_HANDLER = """ + _PyErr_Format(tstate, PyExc_SystemError, + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); + JUMP_TO_LABEL(error); +""" def generate_tier1( - filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool + filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool, tail_call_mode: bool ) -> None: write_header(__file__, filenames, outfile) - outfile.write( - f""" + outfile.write(""" #ifdef TIER_TWO #error "This file is for Tier 1 only" #endif #define TIER_ONE 1 - +""") + generate_tier1_tailcall_metadata(analysis, outfile, lines) + outfile.write(f""" #ifndef Py_TAIL_CALL_INTERP - #if !USE_COMPUTED_GOTOS dispatch_opcode: switch (opcode) #endif {{ +#endif /* Py_TAIL_CALL_INTERP */ {INSTRUCTION_START_MARKER} """ ) - generate_tier1_cases(analysis, outfile, lines) + generate_tier1_cases(analysis, outfile, lines, tail_call_mode) outfile.write(f""" {INSTRUCTION_END_MARKER} +#ifndef Py_TAIL_CALL_INTERP #if USE_COMPUTED_GOTOS _unknown_opcode: #else @@ -212,12 +183,7 @@ def generate_tier1( /* Tell C compilers not to hold the opcode variable in the loop. next_instr points the current instruction without TARGET(). */ opcode = next_instr->op.code; - _PyErr_Format(tstate, PyExc_SystemError, - "%U:%d: unknown opcode %d", - _PyFrame_GetCode(frame)->co_filename, - PyUnstable_InterpreterFrame_GetLine(frame), - opcode); - goto error; + {UNKNOWN_OPCODE_HANDLER} }} @@ -233,26 +199,119 @@ def generate_tier1( outfile.write(f"{LABEL_END_MARKER}\n") outfile.write(FOOTER) + + def generate_tier1_labels( analysis: Analysis, emitter: Emitter ) -> None: emitter.emit("\n") + # Emit tail-callable labels as function defintions for name, label in analysis.labels.items(): - emitter.emit(f"{name}:\n") + emitter.emit(f"LABEL({name})\n") emitter.emit_label(label) emitter.emit("\n") emitter.emit("\n") -def generate_tier1_cases( + + +def function_proto(name: str) -> str: + return f"Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_{name}(TAIL_CALL_PARAMS)" + + +def generate_tier1_tailcall_metadata( analysis: Analysis, outfile: TextIO, lines: bool ) -> None: + + out = CWriter(outfile, 0, lines) + out.emit("#ifdef Py_TAIL_CALL_INTERP\n") + out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256];\n") + out.emit("\n") + + # Emit function prototypes for labels. + for name in analysis.labels: + out.emit(f"{function_proto(name)};\n") + out.emit("\n") + + # Emit function prototypes for opcode handlers. + for name in sorted(analysis.instructions.keys()): + out.emit(f"{function_proto(name)};\n") + out.emit("\n") + + # Emit unknown opcode handler. + out.emit(function_proto("UNKNOWN_OPCODE")) + out.emit(" {\n") + out.emit("int opcode = next_instr->op.code;\n") + out.emit(UNKNOWN_OPCODE_HANDLER) + out.emit("}\n") + out.emit("\n") + + # Emit the dispatch table. + out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256] = {\n") + for name in sorted(analysis.instructions.keys()): + out.emit(f"[{name}] = _TAIL_CALL_{name},\n") + named_values = analysis.opmap.values() + for rest in range(256): + if rest not in named_values: + out.emit(f"[{rest}] = _TAIL_CALL_UNKNOWN_OPCODE,\n") + out.emit("};\n") + outfile.write("#endif /* Py_TAIL_CALL_INTERP */\n") + +def generate_tier1_cases( + analysis: Analysis, outfile: TextIO, lines: bool, tail_call_mode: bool +) -> None: out = CWriter(outfile, 2, lines) emitter = Emitter(out) out.emit("\n") for name, inst in sorted(analysis.instructions.items()): out.emit("\n") out.emit(f"TARGET({name}) {{\n") - write_single_inst(out, emitter, name, inst, uses_this) + if tail_call_mode: + out.emit(f"#ifdef Py_TAIL_CALL_INTERP\n") + out.emit(f"int opcode = next_instr->op.code;\n") + out.emit(f"(void)(opcode);\n") + out.emit(f"#endif /* Py_TAIL_CALL_INTERP */\n") + needs_this, only_for_tail_call = uses_this(inst) + if not tail_call_mode: + if only_for_tail_call and needs_this: + only_for_tail_call = needs_this = False + unused_guard = "(void)this_instr;\n" + if inst.properties.needs_prev: + out.emit(f"_Py_CODEUNIT* const prev_instr = frame->instr_ptr;\n") + + if needs_this and not inst.is_target: + if only_for_tail_call: + out.emit("#ifdef Py_TAIL_CALL_INTERP\n") + out.emit(f"_Py_CODEUNIT* const this_instr = next_instr;\n") + out.emit(unused_guard) + if only_for_tail_call: + out.emit("#endif /* Py_TAIL_CALL_INTERP */\n") + if not inst.properties.no_save_ip: + out.emit(f"frame->instr_ptr = next_instr;\n") + + out.emit(f"next_instr += {inst.size};\n") + out.emit(f"INSTRUCTION_STATS({name});\n") + if inst.is_target: + out.emit(f"PREDICTED_{name}:;\n") + if needs_this: + out.emit(f"_Py_CODEUNIT* const this_instr = next_instr - {inst.size};\n") + out.emit(unused_guard) + if inst.properties.uses_opcode: + out.emit(f"opcode = {name};\n") + if inst.family is not None: + out.emit( + f"static_assert({inst.family.size} == {inst.size-1}" + ', "incorrect cache size");\n' + ) + declare_variables(inst, out) + offset = 1 # The instruction itself + stack = Stack() + for part in inst.parts: + # Only emit braces if more than one uop + insert_braces = len([p for p in inst.parts if isinstance(p, Uop)]) > 1 + offset, stack = write_uop(part, emitter, offset, stack, inst, insert_braces) + out.start_line() + + stack.flush(out) if not inst.parts[-1].properties.always_exits: out.emit("DISPATCH();\n") out.start_line() @@ -279,11 +338,11 @@ def generate_tier1_cases( def generate_tier1_from_files( - filenames: list[str], outfilename: str, lines: bool + filenames: list[str], outfilename: str, lines: bool, tail_call_mode: bool = False ) -> None: data = analyze_files(filenames) with open(outfilename, "w") as outfile: - generate_tier1(filenames, data, outfile, lines) + generate_tier1(filenames, data, outfile, lines, tail_call_mode) if __name__ == "__main__": @@ -292,4 +351,4 @@ def generate_tier1_from_files( args.input.append(DEFAULT_INPUT) data = analyze_files(args.input) with open(args.output, "w") as outfile: - generate_tier1(args.input, data, outfile, args.emit_line_directives) + generate_tier1(args.input, data, outfile, args.emit_line_directives, True) diff --git a/Tools/cases_generator/tier1_tail_call_generator.py b/Tools/cases_generator/tier1_tail_call_generator.py deleted file mode 100644 index 1e1993d17ff62d..00000000000000 --- a/Tools/cases_generator/tier1_tail_call_generator.py +++ /dev/null @@ -1,279 +0,0 @@ -"""Generate the main interpreter tail call handlers. -Reads the instruction definitions from bytecodes.c. -Writes the cases to generated_tail_call_handlers.c.h, which is #included in ceval.c. -""" - -import argparse - -from typing import TextIO - -from generators_common import ( - ROOT, - write_header, - CWriter, - Emitter, - TokenIterator, - emit_to, - always_true, -) - -from analyzer import ( - Analysis, - Instruction, - analyze_files, - Uop, - Label, -) - -from tier1_generator import ( - write_single_inst, - generate_tier1_labels, - INSTRUCTION_START_MARKER, - INSTRUCTION_END_MARKER, -) - -from lexer import Token - -from stack import Storage - -DEFAULT_INPUT = ROOT / "Python/bytecodes.c" -DEFAULT_OUTPUT = ROOT / "Python/generated_tail_call_handlers.c.h" -DEFAULT_LABELS_OUTPUT = ROOT / "Python/generated_tail_call_labels.c.h" - -PRELUDE = """ -#ifndef Py_TAIL_CALL_INTERP - #error "This file is for tail-calling interpreter only." -#endif -#define TIER_ONE 1 -""" -FOOTER = "#undef TIER_ONE\n" - -class TailCallEmitter(Emitter): - - def __init__(self, out: CWriter, analysis: Analysis): - super().__init__(out) - self.analysis = analysis - - def deopt_if( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: Uop | Label, - storage: Storage, - inst: Instruction | None, - ) -> bool: - self.out.start_line() - self.out.emit("if (") - lparen = next(tkn_iter) - assert lparen.kind == "LPAREN" - first_tkn = tkn_iter.peek() - emit_to(self.out, tkn_iter, "RPAREN") - self.emit(") {\n") - next(tkn_iter) # Semi colon - assert inst is not None - assert inst.family is not None - family_name = inst.family.name - self.emit(f"UPDATE_MISS_STATS({family_name});\n") - self.emit(f"assert(_PyOpcode_Deopt[opcode] == ({family_name}));\n") - # Note it's not TAIL_CALL_ARGS, it passes this_instr instead of next_instr. - self.emit(f"Py_MUSTTAIL return _TAIL_CALL_{family_name}(frame, stack_pointer, tstate, this_instr, oparg);\n") - self.emit("}\n") - return not always_true(first_tkn) - - - exit_if = deopt_if - -class TailCallLabelsEmitter(Emitter): - def __init__(self, out: CWriter): - super().__init__(out) - self._replacers = { - 'goto': self.goto, - } - - def goto( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: Uop | Label, - storage: Storage, - inst: Instruction | None, - ) -> bool: - # Only labels need to replace their gotos with tail calls. - # We can't do this in normal code due to GCC's escape analysis - # complaining. - name = next(tkn_iter) - next(tkn_iter) - assert name.kind == "IDENTIFIER" - self.out.start_line() - self.emit(f"TAIL_CALL({name.text});\n") - return True - - -class TailCallCevalLabelsEmitter(Emitter): - def __init__(self, out: CWriter): - super().__init__(out) - self._replacers = { - 'DISPATCH': self.dispatch, - } - - def dispatch( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: Uop | Label, - storage: Storage, - inst: Instruction | None, - ) -> bool: - # Replace DISPATCH with _TAIL_CALL_entry(...) - next(tkn_iter) - next(tkn_iter) - next(tkn_iter) - self.out.start_line() - self.emit("return _TAIL_CALL_entry(frame, stack_pointer, tstate, next_instr, 0);\n") - return True - - -def function_proto(name: str) -> str: - return f"Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_{name}(TAIL_CALL_PARAMS)" - -def generate_label_handlers( - analysis: Analysis, outfile: TextIO, lines: bool -) -> None: - out = CWriter(outfile, 0, lines) - emitter = TailCallLabelsEmitter(out) - emitter.emit("\n") - for name in analysis.labels: - emitter.emit(f"{function_proto(name)};\n") - emitter.emit("\n") - out.emit(f"{INSTRUCTION_START_MARKER}\n") - for name, label in analysis.labels.items(): - emitter.emit(f"{function_proto(name)}\n") - emitter.emit_label(label) - emitter.emit("\n") - emitter.emit("\n") - - -def uses_this(inst: Instruction) -> bool: - if inst.properties.needs_this: - return True - for uop in inst.parts: - if not isinstance(uop, Uop): - continue - for cache in uop.caches: - if cache.name != "unused": - return True - for tkn in uop.body: - if (tkn.kind == "IDENTIFIER" - and (tkn.text in {"DEOPT_IF", "EXIT_IF", "GO_TO_INSTRUCTION"})): - return True - return False - -def generate_tier1( - filenames: list[str], analysis: Analysis, outfile: TextIO, labels_outfile: TextIO, lines: bool -) -> None: - # Write labels required for main ceval - write_header(__file__, filenames, labels_outfile) - labels_outfile.write(PRELUDE) - out = CWriter(labels_outfile, 2, lines) - emitter = TailCallCevalLabelsEmitter(out) - generate_tier1_labels(analysis, emitter) - labels_outfile.write(FOOTER) - - write_header(__file__, filenames, outfile) - outfile.write(PRELUDE) - out = CWriter(outfile, 0, lines) - out.emit("static inline PyObject *_TAIL_CALL_entry(TAIL_CALL_PARAMS);\n") - out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256];\n"); - - generate_label_handlers(analysis, outfile, lines) - - emitter = TailCallEmitter(out, analysis) - for name, inst in sorted(analysis.instructions.items()): - out.emit("\n") - out.emit(function_proto(name)) - out.emit(" {\n") - # We wrap this with a block to signal to GCC that the local variables - # are dead at the tail call site. - # Otherwise, GCC 15's escape analysis may think there are - # escaping locals. - # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=118430#c1 - out.emit("{\n") - write_single_inst(out, emitter, name, inst, uses_this, is_in_tail_call=True) - out.emit("}\n") - if not inst.parts[-1].properties.always_exits: - out.emit("DISPATCH();\n") - # Note: this produces 2 jumps, but a tail call directly - # at the branch also produces the same. - # Furthermore, this is required to make GCC 15's escape analysis happy - # as written above. - for err_label in analysis.labels.keys(): - out.emit(f"{err_label}:\n") - out.emit(f"TAIL_CALL({err_label});\n") - out.start_line() - out.emit("}\n") - - out.emit(f"{INSTRUCTION_END_MARKER}\n") - - # Emit unknown opcode handler. - out.emit(function_proto("UNKNOWN_OPCODE")) - out.emit(" {\n") - out.emit("{\n") - out.emit("int opcode = next_instr->op.code;\n") - out.emit(""" -_PyErr_Format(tstate, PyExc_SystemError, - "%U:%d: unknown opcode %d", - _PyFrame_GetCode(frame)->co_filename, - PyUnstable_InterpreterFrame_GetLine(frame), - opcode); -""") - out.emit("}\n") - out.emit("TAIL_CALL(error);\n") - out.emit("}\n") - - out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256] = {\n") - for name in sorted(analysis.instructions.keys()): - out.emit(f"[{name}] = _TAIL_CALL_{name},\n") - named_values = analysis.opmap.values() - for rest in range(256): - if rest not in named_values: - out.emit(f"[{rest}] = _TAIL_CALL_UNKNOWN_OPCODE,\n") - out.emit("};\n") - outfile.write(FOOTER) - -def generate_tier1_from_files( - filenames: list[str], out_filename: str, out_label_filename: str, lines: bool -) -> None: - data = analyze_files(filenames) - with open(out_filename, "w") as outfile, open(out_label_filename, "w") as labels_outfile: - generate_tier1(filenames, data, outfile, labels_outfile, lines) - - -arg_parser = argparse.ArgumentParser( - description="Generate the code for the interpreter switch.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, -) - -arg_parser.add_argument( - "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT -) - -arg_parser.add_argument( - "-lo", "--labels-output", type=str, help="Generated labels", default=DEFAULT_LABELS_OUTPUT -) - -arg_parser.add_argument( - "-l", "--emit-line-directives", help="Emit #line directives", action="store_true" -) - -arg_parser.add_argument( - "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" -) - - -if __name__ == "__main__": - args = arg_parser.parse_args() - if len(args.input) == 0: - args.input.append(DEFAULT_INPUT) - data = analyze_files(args.input) - with open(args.output, "w") as outfile, open(args.labels_output, 'w') as labels_outfile: - generate_tier1(args.input, data, outfile, labels_outfile, args.emit_line_directives) From c65688d9af5aaca2a7a78a2b280ac3e93808f01e Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 4 Feb 2025 04:43:44 +0800 Subject: [PATCH 101/110] LINT --- Python/generated_cases.c.h | 26 +++++++++++------------- Tools/cases_generator/tier1_generator.py | 16 +++++++-------- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ee4e09f449fab4..7f3deaf251d5e4 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -244,13 +244,12 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { int opcode = next_instr->op.code; - - _PyErr_Format(tstate, PyExc_SystemError, - "%U:%d: unknown opcode %d", - _PyFrame_GetCode(frame)->co_filename, - PyUnstable_InterpreterFrame_GetLine(frame), - opcode); - JUMP_TO_LABEL(error); + _PyErr_Format(tstate, PyExc_SystemError, + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); +JUMP_TO_LABEL(error); } static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { @@ -512,7 +511,7 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [234] = _TAIL_CALL_UNKNOWN_OPCODE, }; #endif /* Py_TAIL_CALL_INTERP */ - + #ifndef Py_TAIL_CALL_INTERP #if !USE_COMPUTED_GOTOS dispatch_opcode: @@ -11862,13 +11861,12 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { /* Tell C compilers not to hold the opcode variable in the loop. next_instr points the current instruction without TARGET(). */ opcode = next_instr->op.code; - _PyErr_Format(tstate, PyExc_SystemError, - "%U:%d: unknown opcode %d", - _PyFrame_GetCode(frame)->co_filename, - PyUnstable_InterpreterFrame_GetLine(frame), - opcode); - JUMP_TO_LABEL(error); + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); +JUMP_TO_LABEL(error); } diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 2f5042bec9721e..8b6bce725a2bb5 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -140,13 +140,13 @@ def uses_this(inst: Instruction) -> tuple[bool, bool]: return False, False -UNKNOWN_OPCODE_HANDLER = """ - _PyErr_Format(tstate, PyExc_SystemError, - "%U:%d: unknown opcode %d", - _PyFrame_GetCode(frame)->co_filename, - PyUnstable_InterpreterFrame_GetLine(frame), - opcode); - JUMP_TO_LABEL(error); +UNKNOWN_OPCODE_HANDLER ="""\ +_PyErr_Format(tstate, PyExc_SystemError, + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); +JUMP_TO_LABEL(error); """ def generate_tier1( @@ -160,7 +160,7 @@ def generate_tier1( #define TIER_ONE 1 """) generate_tier1_tailcall_metadata(analysis, outfile, lines) - outfile.write(f""" + outfile.write(f""" #ifndef Py_TAIL_CALL_INTERP #if !USE_COMPUTED_GOTOS dispatch_opcode: From 22ea2e59c4caa5a36a07c9e50840dfc38a963b6f Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 4 Feb 2025 11:24:04 +0800 Subject: [PATCH 102/110] Simplify --- Python/ceval.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 4655421b94ee8f..9a0c0bdfe82f90 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -866,14 +866,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, 0); #else goto start_frame; +# include "generated_cases.c.h" #endif -#ifndef Py_TAIL_CALL_INTERP -# include "generated_cases.c.h" -#endif - - - #ifdef _Py_TIER2 From ddbddc1858c8bef5a950f424dc7709d822666a28 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 4 Feb 2025 15:15:14 +0800 Subject: [PATCH 103/110] lint --- Python/ceval.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/ceval.c b/Python/ceval.c index 9a0c0bdfe82f90..b96fdfb5f3d947 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -866,7 +866,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, 0); #else goto start_frame; -# include "generated_cases.c.h" +# include "generated_cases.c.h" #endif From 0190825a452d7346f4f294d34602316ebecbf38d Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 5 Feb 2025 00:50:26 +0800 Subject: [PATCH 104/110] Fix bugs from upstream --- Lib/test/test_generated_cases.py | 10 +++++++--- Python/bytecodes.c | 6 ++---- Python/generated_cases.c.h | 15 +++++++-------- Tools/cases_generator/generators_common.py | 15 ++++++++++++--- 4 files changed, 28 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index d140c3296e17c3..2af2d8f4e33c94 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -672,7 +672,7 @@ def test_suppress_dispatch(self): goto somewhere; } - somewhere: + LABEL(somewhere) { } @@ -1837,14 +1837,14 @@ def test_spilled_label(self): """ output = """ - one: + LABEL(one) { /* STACK SPILLED */ stack_pointer = _PyFrame_GetStackPointer(frame); goto two; } - two: + LABEL(two) { _PyFrame_SetStackPointer(frame, stack_pointer); goto one; @@ -1896,14 +1896,18 @@ def test_multiple_labels(self): LABEL(my_label_1) { // Comment + _PyFrame_SetStackPointer(frame, stack_pointer); do_thing1(); + stack_pointer = _PyFrame_GetStackPointer(frame); goto my_label_2; } LABEL(my_label_2) { // Comment + _PyFrame_SetStackPointer(frame, stack_pointer); do_thing2(); + stack_pointer = _PyFrame_GetStackPointer(frame); goto my_label_1; } """ diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f5d6e211b28f8b..84d0da86f52869 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5309,8 +5309,7 @@ dummy_func( RELOAD_STACK(); #ifdef Py_TAIL_CALL_INTERP int opcode; - (void)(opcode); -#endif +#endif DISPATCH(); } @@ -5360,8 +5359,7 @@ dummy_func( RELOAD_STACK(); #ifdef Py_TAIL_CALL_INTERP int opcode; - (void)(opcode); -#endif +#endif DISPATCH(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7857d4c80231fd..88fa838fe0bc36 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4744,6 +4744,7 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); JUMP_TO_LABEL(exception_unwind); } stack_pointer[-3] = none; @@ -5465,6 +5466,7 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); JUMP_TO_LABEL(exception_unwind); } stack_pointer += -2; @@ -10049,6 +10051,7 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { _PyFrame_SetStackPointer(frame, stack_pointer); monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); JUMP_TO_LABEL(exception_unwind); } JUMP_TO_LABEL(error); @@ -10099,6 +10102,7 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); JUMP_TO_LABEL(exception_unwind); DISPATCH(); } @@ -11971,6 +11975,7 @@ JUMP_TO_LABEL(error); _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_MonitorRaise(tstate, frame, next_instr-1); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); JUMP_TO_LABEL(exception_unwind); } @@ -12028,10 +12033,7 @@ JUMP_TO_LABEL(error); stack_pointer = _PyFrame_GetStackPointer(frame); #ifdef Py_TAIL_CALL_INTERP int opcode; - _PyFrame_SetStackPointer(frame, stack_pointer); - (void)(opcode); - stack_pointer = _PyFrame_GetStackPointer(frame); - #endif + #endif DISPATCH(); } @@ -12084,10 +12086,7 @@ JUMP_TO_LABEL(error); stack_pointer = _PyFrame_GetStackPointer(frame); #ifdef Py_TAIL_CALL_INTERP int opcode; - _PyFrame_SetStackPointer(frame, stack_pointer); - (void)(opcode); - stack_pointer = _PyFrame_GetStackPointer(frame); - #endif + #endif DISPATCH(); } diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 0582fc2489a073..30ab0e0385b0bd 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -404,7 +404,7 @@ def stack_pointer( self.emit(tkn) return True - def goto_label(self, goto: Token, label: Token, storage: Storage) -> None: + def goto_label(self, goto: Token, label: Token, storage: Storage, wrap_paren: bool = False) -> None: if label.text not in self.labels: print(self.labels.keys()) raise analysis_error(f"Label '{label.text}' does not exist", label) @@ -415,7 +415,11 @@ def goto_label(self, goto: Token, label: Token, storage: Storage) -> None: elif storage.spilled: raise analysis_error("Cannot jump from spilled label without reloading the stack pointer", goto) self.out.emit(goto) + if wrap_paren: + self.out.emit("(") self.out.emit(label) + if wrap_paren: + self.out.emit(")") def emit_save(self, storage: Storage) -> None: storage.save(self.out) @@ -607,7 +611,7 @@ def _emit_block( elif tkn.kind == "GOTO": label_tkn = next(tkn_iter) self.goto_label(tkn, label_tkn, storage) - reachable = False; + reachable = False elif tkn.kind == "IDENTIFIER": if tkn.text in self._replacers: if not self._replacers[tkn.text](tkn, tkn_iter, uop, storage, inst): @@ -623,8 +627,13 @@ def _emit_block( self._print_storage(storage) reachable = False if tkn.text.startswith("JUMP_TO_LABEL"): + next(tkn_iter) + label_tkn = next(tkn_iter) + next(tkn_iter) + self.goto_label(tkn, label_tkn, storage, wrap_paren=True) reachable = False - self.out.emit(tkn) + else: + self.out.emit(tkn) elif tkn.kind == "IF": self.out.emit(tkn) if_reachable, rbrace, storage = self._emit_if(tkn_iter, uop, storage, inst) From bbea58fc985a66f8d4063d59fb5a23e931ea6b17 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 5 Feb 2025 00:53:25 +0800 Subject: [PATCH 105/110] Update optimizer_generator.py --- Tools/cases_generator/optimizer_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 6c33debd58e1fe..3f162e72aa7303 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -112,7 +112,7 @@ def emit_save(self, storage: Storage) -> None: def emit_reload(self, storage: Storage) -> None: pass - def goto_label(self, goto: Token, label: Token, storage: Storage) -> None: + def goto_label(self, goto: Token, label: Token, storage: Storage, wrap_paren: bool = False) -> None: self.out.emit(goto) self.out.emit(label) From 98e5e68ae66e15057bd2eddc3927ab3bd2225906 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 6 Feb 2025 20:03:40 +0800 Subject: [PATCH 106/110] Update ceval_macros.h --- Python/ceval_macros.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index ddf38b237e8453..44bb52a09aab5f 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -79,7 +79,7 @@ # define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) Py_PRESERVE_NONE_CC typedef PyObject* (*py_tail_call_funcptr)(TAIL_CALL_PARAMS); -# define TARGET(op) PyObject *_TAIL_CALL_##op(TAIL_CALL_PARAMS) +# define TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_##op(TAIL_CALL_PARAMS) # define DISPATCH_GOTO() \ do { \ Py_MUSTTAIL return (INSTRUCTION_TABLE[opcode])(TAIL_CALL_ARGS); \ From 45161c037e610031f74bf6a4da763523f8eed82c Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 6 Feb 2025 21:59:42 +0800 Subject: [PATCH 107/110] Address review --- Lib/test/test_generated_cases.py | 216 ++- Python/bytecodes.c | 32 +- Python/ceval.c | 1 + Python/generated_cases.c.h | 1699 +++++------------- Python/opcode_targets.h | 505 ++++++ Tools/cases_generator/generators_common.py | 37 +- Tools/cases_generator/optimizer_generator.py | 2 +- Tools/cases_generator/target_generator.py | 40 + Tools/cases_generator/tier1_generator.py | 76 +- Tools/cases_generator/tier2_generator.py | 1 - 10 files changed, 1198 insertions(+), 1411 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 2af2d8f4e33c94..aa0899ecaf860f 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -304,6 +304,10 @@ def test_inst_no_args(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -322,6 +326,10 @@ def test_inst_one_pop(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -343,6 +351,10 @@ def test_inst_one_push(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -365,6 +377,10 @@ def test_inst_one_push_one_pop(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -388,6 +404,10 @@ def test_binary_op(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -414,6 +434,10 @@ def test_overlap(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -442,10 +466,14 @@ def test_predictions(self): """ output = """ TARGET(OP1) { + int opcode = OP1; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP1); PREDICTED_OP1:; + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; _PyStackRef res; res = Py_None; stack_pointer[-1] = res; @@ -453,6 +481,10 @@ def test_predictions(self): } TARGET(OP3) { + int opcode = OP3; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP3); @@ -485,6 +517,10 @@ def test_sync_sp(self): """ output = """ TARGET(A) { + int opcode = A; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(A); @@ -502,6 +538,10 @@ def test_sync_sp(self): } TARGET(B) { + int opcode = B; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(B); @@ -539,6 +579,10 @@ def test_error_if_plain(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -558,6 +602,10 @@ def test_error_if_plain_with_comment(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -581,6 +629,10 @@ def test_error_if_pop(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -612,6 +664,10 @@ def test_error_if_pop_with_result(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -639,6 +695,8 @@ def test_cache_effect(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -666,10 +724,14 @@ def test_suppress_dispatch(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - goto somewhere; + JUMP_TO_LABEL(somewhere); } LABEL(somewhere) @@ -697,6 +759,8 @@ def test_macro_instruction(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(OP); @@ -734,6 +798,8 @@ def test_macro_instruction(self): } TARGET(OP1) { + int opcode = OP1; + (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -752,6 +818,10 @@ def test_macro_instruction(self): } TARGET(OP3) { + int opcode = OP3; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(OP3); @@ -783,6 +853,10 @@ def test_unused_caches(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(OP); @@ -805,6 +879,10 @@ def test_pseudo_instruction_no_flags(self): """ output = """ TARGET(OP1) { + int opcode = OP1; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP1); @@ -824,6 +902,10 @@ def test_pseudo_instruction_with_flags(self): """ output = """ TARGET(OP1) { + int opcode = OP1; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP1); @@ -846,6 +928,10 @@ def test_pseudo_instruction_as_sequence(self): """ output = """ TARGET(OP1) { + int opcode = OP1; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP1); @@ -853,6 +939,10 @@ def test_pseudo_instruction_as_sequence(self): } TARGET(OP2) { + int opcode = OP2; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP2); @@ -870,6 +960,10 @@ def test_array_input(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -893,6 +987,10 @@ def test_array_output(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -921,6 +1019,10 @@ def test_array_input_output(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -945,6 +1047,10 @@ def test_array_error_if(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -971,6 +1077,10 @@ def test_cond_effect(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1012,6 +1122,10 @@ def test_macro_cond_effect(self): """ output = """ TARGET(M) { + int opcode = M; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); @@ -1056,6 +1170,10 @@ def test_macro_push_push(self): """ output = """ TARGET(M) { + int opcode = M; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); @@ -1089,6 +1207,10 @@ def test_override_inst(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1110,6 +1232,10 @@ def test_override_op(self): """ output = """ TARGET(M) { + int opcode = M; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); @@ -1127,6 +1253,10 @@ def test_annotated_inst(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1145,6 +1275,10 @@ def test_annotated_op(self): """ output = """ TARGET(M) { + int opcode = M; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); @@ -1181,6 +1315,10 @@ def test_array_of_one(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1203,6 +1341,10 @@ def test_pointer_to_stackref(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1241,6 +1383,10 @@ def test_unused_named_values(self): """ output = """ TARGET(INST) { + int opcode = INST; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INST); @@ -1267,6 +1413,10 @@ def test_used_unused_used(self): """ output = """ TARGET(TEST) { + int opcode = TEST; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(TEST); @@ -1307,6 +1457,10 @@ def test_unused_used_used(self): """ output = """ TARGET(TEST) { + int opcode = TEST; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(TEST); @@ -1346,6 +1500,10 @@ def test_flush(self): """ output = """ TARGET(TEST) { + int opcode = TEST; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(TEST); @@ -1394,6 +1552,10 @@ def test_pop_on_error_peeks(self): """ output = """ TARGET(TEST) { + int opcode = TEST; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(TEST); @@ -1442,6 +1604,10 @@ def test_push_then_error(self): output = """ TARGET(TEST) { + int opcode = TEST; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(TEST); @@ -1483,6 +1649,10 @@ def test_error_if_true(self): """ output = """ TARGET(OP1) { + int opcode = OP1; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP1); @@ -1490,6 +1660,10 @@ def test_error_if_true(self): } TARGET(OP2) { + int opcode = OP2; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP2); @@ -1547,6 +1721,10 @@ def test_stack_save_reload(self): output = """ TARGET(BALANCED) { + int opcode = BALANCED; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BALANCED); @@ -1567,6 +1745,10 @@ def test_stack_reload_only(self): output = """ TARGET(BALANCED) { + int opcode = BALANCED; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BALANCED); @@ -1588,6 +1770,10 @@ def test_stack_save_only(self): output = """ TARGET(BALANCED) { + int opcode = BALANCED; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BALANCED); @@ -1608,6 +1794,10 @@ def test_instruction_size_macro(self): output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1643,6 +1833,10 @@ def test_escaping_call_next_to_cmacro(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1677,6 +1871,10 @@ def test_pystackref_frompyobject_new_next_to_cmacro(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1708,6 +1906,10 @@ def test_pop_input(self): """ output = """ TARGET(OP) { + int opcode = OP; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1816,9 +2018,9 @@ def test_complex_label(self): do_thing(); stack_pointer = _PyFrame_GetStackPointer(frame); if (complex) { - goto other_label; + JUMP_TO_LABEL(other_label); } - goto other_label2; + JUMP_TO_LABEL(other_label2); } """ self.run_cases_test(input, output) @@ -1841,13 +2043,13 @@ def test_spilled_label(self): { /* STACK SPILLED */ stack_pointer = _PyFrame_GetStackPointer(frame); - goto two; + JUMP_TO_LABEL(two); } LABEL(two) { _PyFrame_SetStackPointer(frame, stack_pointer); - goto one; + JUMP_TO_LABEL(one); } """ self.run_cases_test(input, output) @@ -1899,7 +2101,7 @@ def test_multiple_labels(self): _PyFrame_SetStackPointer(frame, stack_pointer); do_thing1(); stack_pointer = _PyFrame_GetStackPointer(frame); - goto my_label_2; + JUMP_TO_LABEL(my_label_2); } LABEL(my_label_2) @@ -1908,7 +2110,7 @@ def test_multiple_labels(self): _PyFrame_SetStackPointer(frame, stack_pointer); do_thing2(); stack_pointer = _PyFrame_GetStackPointer(frame); - goto my_label_1; + JUMP_TO_LABEL(my_label_1); } """ self.run_cases_test(input, output) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 84d0da86f52869..c3024f7f98f28c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1100,7 +1100,7 @@ dummy_func( if (err) { assert(oparg == 0); monitor_reraise(tstate, frame, this_instr); - JUMP_TO_LABEL(exception_unwind); + goto exception_unwind; } ERROR_IF(true, error); } @@ -1366,7 +1366,7 @@ dummy_func( assert(exc && PyExceptionInstance_Check(exc)); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); - JUMP_TO_LABEL(exception_unwind); + goto exception_unwind; } tier1 inst(END_ASYNC_FOR, (awaitable_st, exc_st -- )) { @@ -1381,7 +1381,7 @@ dummy_func( Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); - JUMP_TO_LABEL(exception_unwind); + goto exception_unwind; } } @@ -1401,7 +1401,7 @@ dummy_func( else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); monitor_reraise(tstate, frame, this_instr); - JUMP_TO_LABEL(exception_unwind); + goto exception_unwind; } } @@ -4849,7 +4849,7 @@ dummy_func( tstate, frame, this_instr, prev_instr); if (original_opcode < 0) { next_instr = this_instr+1; - JUMP_TO_LABEL(error); + goto error; } next_instr = frame->instr_ptr; if (next_instr != this_instr) { @@ -5214,22 +5214,22 @@ dummy_func( label(pop_4_error) { STACK_SHRINK(4); - JUMP_TO_LABEL(error); + goto error; } label(pop_3_error) { STACK_SHRINK(3); - JUMP_TO_LABEL(error); + goto error; } label(pop_2_error) { STACK_SHRINK(2); - JUMP_TO_LABEL(error); + goto error; } label(pop_1_error) { STACK_SHRINK(1); - JUMP_TO_LABEL(error); + goto error; } label(error) { @@ -5252,7 +5252,7 @@ dummy_func( } } _PyEval_MonitorRaise(tstate, frame, next_instr-1); - JUMP_TO_LABEL(exception_unwind); + goto exception_unwind; } spilled label(exception_unwind) { @@ -5270,7 +5270,7 @@ dummy_func( PyStackRef_XCLOSE(ref); } monitor_unwind(tstate, frame, next_instr-1); - JUMP_TO_LABEL(exit_unwind); + goto exit_unwind; } assert(STACK_LEVEL() >= level); _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; @@ -5283,7 +5283,7 @@ dummy_func( int frame_lasti = _PyInterpreterFrame_LASTI(frame); PyObject *lasti = PyLong_FromLong(frame_lasti); if (lasti == NULL) { - JUMP_TO_LABEL(exception_unwind); + goto exception_unwind; } _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(lasti)); } @@ -5298,7 +5298,7 @@ dummy_func( int err = monitor_handled(tstate, frame, next_instr, exc); if (err < 0) { - JUMP_TO_LABEL(exception_unwind); + goto exception_unwind; } /* Resume normal execution */ #ifdef LLTRACE @@ -5330,13 +5330,13 @@ dummy_func( } next_instr = frame->instr_ptr; RELOAD_STACK(); - JUMP_TO_LABEL(error); + goto error; } spilled label(start_frame) { int too_deep = _Py_EnterRecursivePy(tstate); if (too_deep) { - JUMP_TO_LABEL(exit_unwind); + goto exit_unwind; } next_instr = frame->instr_ptr; @@ -5345,7 +5345,7 @@ dummy_func( int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); frame->lltrace = lltrace; if (lltrace < 0) { - JUMP_TO_LABEL(exit_unwind); + goto exit_unwind; } } #endif diff --git a/Python/ceval.c b/Python/ceval.c index d82a11110d7f93..5e834883f355e1 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -769,6 +769,7 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) #ifdef Py_TAIL_CALL_INTERP +#include "opcode_targets.h" #include "generated_cases.c.h" #endif diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 88fa838fe0bc36..828954efe065ef 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7,510 +7,6 @@ #error "This file is for Tier 1 only" #endif #define TIER_ONE 1 -#ifdef Py_TAIL_CALL_INTERP -static py_tail_call_funcptr INSTRUCTION_TABLE[256]; - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_4_error(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_3_error(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_MORTAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); - -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { - int opcode = next_instr->op.code; - _PyErr_Format(tstate, PyExc_SystemError, - "%U:%d: unknown opcode %d", - _PyFrame_GetCode(frame)->co_filename, - PyUnstable_InterpreterFrame_GetLine(frame), - opcode); -JUMP_TO_LABEL(error); -} - -static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { - [BINARY_OP] = _TAIL_CALL_BINARY_OP, - [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, - [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, - [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_BINARY_OP_ADD_UNICODE, - [BINARY_OP_EXTEND] = _TAIL_CALL_BINARY_OP_EXTEND, - [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE, - [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT, - [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_BINARY_OP_MULTIPLY_INT, - [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT, - [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_BINARY_OP_SUBTRACT_INT, - [BINARY_SLICE] = _TAIL_CALL_BINARY_SLICE, - [BINARY_SUBSCR] = _TAIL_CALL_BINARY_SUBSCR, - [BINARY_SUBSCR_DICT] = _TAIL_CALL_BINARY_SUBSCR_DICT, - [BINARY_SUBSCR_GETITEM] = _TAIL_CALL_BINARY_SUBSCR_GETITEM, - [BINARY_SUBSCR_LIST_INT] = _TAIL_CALL_BINARY_SUBSCR_LIST_INT, - [BINARY_SUBSCR_STR_INT] = _TAIL_CALL_BINARY_SUBSCR_STR_INT, - [BINARY_SUBSCR_TUPLE_INT] = _TAIL_CALL_BINARY_SUBSCR_TUPLE_INT, - [BUILD_LIST] = _TAIL_CALL_BUILD_LIST, - [BUILD_MAP] = _TAIL_CALL_BUILD_MAP, - [BUILD_SET] = _TAIL_CALL_BUILD_SET, - [BUILD_SLICE] = _TAIL_CALL_BUILD_SLICE, - [BUILD_STRING] = _TAIL_CALL_BUILD_STRING, - [BUILD_TUPLE] = _TAIL_CALL_BUILD_TUPLE, - [CACHE] = _TAIL_CALL_CACHE, - [CALL] = _TAIL_CALL_CALL, - [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT, - [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS, - [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_CALL_BOUND_METHOD_GENERAL, - [CALL_BUILTIN_CLASS] = _TAIL_CALL_CALL_BUILTIN_CLASS, - [CALL_BUILTIN_FAST] = _TAIL_CALL_CALL_BUILTIN_FAST, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS, - [CALL_BUILTIN_O] = _TAIL_CALL_CALL_BUILTIN_O, - [CALL_FUNCTION_EX] = _TAIL_CALL_CALL_FUNCTION_EX, - [CALL_INTRINSIC_1] = _TAIL_CALL_CALL_INTRINSIC_1, - [CALL_INTRINSIC_2] = _TAIL_CALL_CALL_INTRINSIC_2, - [CALL_ISINSTANCE] = _TAIL_CALL_CALL_ISINSTANCE, - [CALL_KW] = _TAIL_CALL_CALL_KW, - [CALL_KW_BOUND_METHOD] = _TAIL_CALL_CALL_KW_BOUND_METHOD, - [CALL_KW_NON_PY] = _TAIL_CALL_CALL_KW_NON_PY, - [CALL_KW_PY] = _TAIL_CALL_CALL_KW_PY, - [CALL_LEN] = _TAIL_CALL_CALL_LEN, - [CALL_LIST_APPEND] = _TAIL_CALL_CALL_LIST_APPEND, - [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, - [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS, - [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O, - [CALL_NON_PY_GENERAL] = _TAIL_CALL_CALL_NON_PY_GENERAL, - [CALL_PY_EXACT_ARGS] = _TAIL_CALL_CALL_PY_EXACT_ARGS, - [CALL_PY_GENERAL] = _TAIL_CALL_CALL_PY_GENERAL, - [CALL_STR_1] = _TAIL_CALL_CALL_STR_1, - [CALL_TUPLE_1] = _TAIL_CALL_CALL_TUPLE_1, - [CALL_TYPE_1] = _TAIL_CALL_CALL_TYPE_1, - [CHECK_EG_MATCH] = _TAIL_CALL_CHECK_EG_MATCH, - [CHECK_EXC_MATCH] = _TAIL_CALL_CHECK_EXC_MATCH, - [CLEANUP_THROW] = _TAIL_CALL_CLEANUP_THROW, - [COMPARE_OP] = _TAIL_CALL_COMPARE_OP, - [COMPARE_OP_FLOAT] = _TAIL_CALL_COMPARE_OP_FLOAT, - [COMPARE_OP_INT] = _TAIL_CALL_COMPARE_OP_INT, - [COMPARE_OP_STR] = _TAIL_CALL_COMPARE_OP_STR, - [CONTAINS_OP] = _TAIL_CALL_CONTAINS_OP, - [CONTAINS_OP_DICT] = _TAIL_CALL_CONTAINS_OP_DICT, - [CONTAINS_OP_SET] = _TAIL_CALL_CONTAINS_OP_SET, - [CONVERT_VALUE] = _TAIL_CALL_CONVERT_VALUE, - [COPY] = _TAIL_CALL_COPY, - [COPY_FREE_VARS] = _TAIL_CALL_COPY_FREE_VARS, - [DELETE_ATTR] = _TAIL_CALL_DELETE_ATTR, - [DELETE_DEREF] = _TAIL_CALL_DELETE_DEREF, - [DELETE_FAST] = _TAIL_CALL_DELETE_FAST, - [DELETE_GLOBAL] = _TAIL_CALL_DELETE_GLOBAL, - [DELETE_NAME] = _TAIL_CALL_DELETE_NAME, - [DELETE_SUBSCR] = _TAIL_CALL_DELETE_SUBSCR, - [DICT_MERGE] = _TAIL_CALL_DICT_MERGE, - [DICT_UPDATE] = _TAIL_CALL_DICT_UPDATE, - [END_ASYNC_FOR] = _TAIL_CALL_END_ASYNC_FOR, - [END_FOR] = _TAIL_CALL_END_FOR, - [END_SEND] = _TAIL_CALL_END_SEND, - [ENTER_EXECUTOR] = _TAIL_CALL_ENTER_EXECUTOR, - [EXIT_INIT_CHECK] = _TAIL_CALL_EXIT_INIT_CHECK, - [EXTENDED_ARG] = _TAIL_CALL_EXTENDED_ARG, - [FORMAT_SIMPLE] = _TAIL_CALL_FORMAT_SIMPLE, - [FORMAT_WITH_SPEC] = _TAIL_CALL_FORMAT_WITH_SPEC, - [FOR_ITER] = _TAIL_CALL_FOR_ITER, - [FOR_ITER_GEN] = _TAIL_CALL_FOR_ITER_GEN, - [FOR_ITER_LIST] = _TAIL_CALL_FOR_ITER_LIST, - [FOR_ITER_RANGE] = _TAIL_CALL_FOR_ITER_RANGE, - [FOR_ITER_TUPLE] = _TAIL_CALL_FOR_ITER_TUPLE, - [GET_AITER] = _TAIL_CALL_GET_AITER, - [GET_ANEXT] = _TAIL_CALL_GET_ANEXT, - [GET_AWAITABLE] = _TAIL_CALL_GET_AWAITABLE, - [GET_ITER] = _TAIL_CALL_GET_ITER, - [GET_LEN] = _TAIL_CALL_GET_LEN, - [GET_YIELD_FROM_ITER] = _TAIL_CALL_GET_YIELD_FROM_ITER, - [IMPORT_FROM] = _TAIL_CALL_IMPORT_FROM, - [IMPORT_NAME] = _TAIL_CALL_IMPORT_NAME, - [INSTRUMENTED_CALL] = _TAIL_CALL_INSTRUMENTED_CALL, - [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX, - [INSTRUMENTED_CALL_KW] = _TAIL_CALL_INSTRUMENTED_CALL_KW, - [INSTRUMENTED_END_FOR] = _TAIL_CALL_INSTRUMENTED_END_FOR, - [INSTRUMENTED_END_SEND] = _TAIL_CALL_INSTRUMENTED_END_SEND, - [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_INSTRUMENTED_FOR_ITER, - [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_INSTRUMENTED_INSTRUCTION, - [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD, - [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD, - [INSTRUMENTED_LINE] = _TAIL_CALL_INSTRUMENTED_LINE, - [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR, - [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_INSTRUMENTED_NOT_TAKEN, - [INSTRUMENTED_POP_ITER] = _TAIL_CALL_INSTRUMENTED_POP_ITER, - [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE, - [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE, - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, - [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE, - [INSTRUMENTED_RESUME] = _TAIL_CALL_INSTRUMENTED_RESUME, - [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_INSTRUMENTED_RETURN_VALUE, - [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_INSTRUMENTED_YIELD_VALUE, - [INTERPRETER_EXIT] = _TAIL_CALL_INTERPRETER_EXIT, - [IS_OP] = _TAIL_CALL_IS_OP, - [JUMP_BACKWARD] = _TAIL_CALL_JUMP_BACKWARD, - [JUMP_BACKWARD_JIT] = _TAIL_CALL_JUMP_BACKWARD_JIT, - [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT, - [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_JUMP_BACKWARD_NO_JIT, - [JUMP_FORWARD] = _TAIL_CALL_JUMP_FORWARD, - [LIST_APPEND] = _TAIL_CALL_LIST_APPEND, - [LIST_EXTEND] = _TAIL_CALL_LIST_EXTEND, - [LOAD_ATTR] = _TAIL_CALL_LOAD_ATTR, - [LOAD_ATTR_CLASS] = _TAIL_CALL_LOAD_ATTR_CLASS, - [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, - [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE, - [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT, - [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT, - [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES, - [LOAD_ATTR_MODULE] = _TAIL_CALL_LOAD_ATTR_MODULE, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, - [LOAD_ATTR_PROPERTY] = _TAIL_CALL_LOAD_ATTR_PROPERTY, - [LOAD_ATTR_SLOT] = _TAIL_CALL_LOAD_ATTR_SLOT, - [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_LOAD_ATTR_WITH_HINT, - [LOAD_BUILD_CLASS] = _TAIL_CALL_LOAD_BUILD_CLASS, - [LOAD_COMMON_CONSTANT] = _TAIL_CALL_LOAD_COMMON_CONSTANT, - [LOAD_CONST] = _TAIL_CALL_LOAD_CONST, - [LOAD_CONST_IMMORTAL] = _TAIL_CALL_LOAD_CONST_IMMORTAL, - [LOAD_CONST_MORTAL] = _TAIL_CALL_LOAD_CONST_MORTAL, - [LOAD_DEREF] = _TAIL_CALL_LOAD_DEREF, - [LOAD_FAST] = _TAIL_CALL_LOAD_FAST, - [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_LOAD_FAST_AND_CLEAR, - [LOAD_FAST_CHECK] = _TAIL_CALL_LOAD_FAST_CHECK, - [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_LOAD_FAST_LOAD_FAST, - [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF, - [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS, - [LOAD_GLOBAL] = _TAIL_CALL_LOAD_GLOBAL, - [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_LOAD_GLOBAL_BUILTIN, - [LOAD_GLOBAL_MODULE] = _TAIL_CALL_LOAD_GLOBAL_MODULE, - [LOAD_LOCALS] = _TAIL_CALL_LOAD_LOCALS, - [LOAD_NAME] = _TAIL_CALL_LOAD_NAME, - [LOAD_SMALL_INT] = _TAIL_CALL_LOAD_SMALL_INT, - [LOAD_SPECIAL] = _TAIL_CALL_LOAD_SPECIAL, - [LOAD_SUPER_ATTR] = _TAIL_CALL_LOAD_SUPER_ATTR, - [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_LOAD_SUPER_ATTR_ATTR, - [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_LOAD_SUPER_ATTR_METHOD, - [MAKE_CELL] = _TAIL_CALL_MAKE_CELL, - [MAKE_FUNCTION] = _TAIL_CALL_MAKE_FUNCTION, - [MAP_ADD] = _TAIL_CALL_MAP_ADD, - [MATCH_CLASS] = _TAIL_CALL_MATCH_CLASS, - [MATCH_KEYS] = _TAIL_CALL_MATCH_KEYS, - [MATCH_MAPPING] = _TAIL_CALL_MATCH_MAPPING, - [MATCH_SEQUENCE] = _TAIL_CALL_MATCH_SEQUENCE, - [NOP] = _TAIL_CALL_NOP, - [NOT_TAKEN] = _TAIL_CALL_NOT_TAKEN, - [POP_EXCEPT] = _TAIL_CALL_POP_EXCEPT, - [POP_ITER] = _TAIL_CALL_POP_ITER, - [POP_JUMP_IF_FALSE] = _TAIL_CALL_POP_JUMP_IF_FALSE, - [POP_JUMP_IF_NONE] = _TAIL_CALL_POP_JUMP_IF_NONE, - [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_POP_JUMP_IF_NOT_NONE, - [POP_JUMP_IF_TRUE] = _TAIL_CALL_POP_JUMP_IF_TRUE, - [POP_TOP] = _TAIL_CALL_POP_TOP, - [PUSH_EXC_INFO] = _TAIL_CALL_PUSH_EXC_INFO, - [PUSH_NULL] = _TAIL_CALL_PUSH_NULL, - [RAISE_VARARGS] = _TAIL_CALL_RAISE_VARARGS, - [RERAISE] = _TAIL_CALL_RERAISE, - [RESERVED] = _TAIL_CALL_RESERVED, - [RESUME] = _TAIL_CALL_RESUME, - [RESUME_CHECK] = _TAIL_CALL_RESUME_CHECK, - [RETURN_GENERATOR] = _TAIL_CALL_RETURN_GENERATOR, - [RETURN_VALUE] = _TAIL_CALL_RETURN_VALUE, - [SEND] = _TAIL_CALL_SEND, - [SEND_GEN] = _TAIL_CALL_SEND_GEN, - [SETUP_ANNOTATIONS] = _TAIL_CALL_SETUP_ANNOTATIONS, - [SET_ADD] = _TAIL_CALL_SET_ADD, - [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_SET_FUNCTION_ATTRIBUTE, - [SET_UPDATE] = _TAIL_CALL_SET_UPDATE, - [STORE_ATTR] = _TAIL_CALL_STORE_ATTR, - [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE, - [STORE_ATTR_SLOT] = _TAIL_CALL_STORE_ATTR_SLOT, - [STORE_ATTR_WITH_HINT] = _TAIL_CALL_STORE_ATTR_WITH_HINT, - [STORE_DEREF] = _TAIL_CALL_STORE_DEREF, - [STORE_FAST] = _TAIL_CALL_STORE_FAST, - [STORE_FAST_LOAD_FAST] = _TAIL_CALL_STORE_FAST_LOAD_FAST, - [STORE_FAST_STORE_FAST] = _TAIL_CALL_STORE_FAST_STORE_FAST, - [STORE_GLOBAL] = _TAIL_CALL_STORE_GLOBAL, - [STORE_NAME] = _TAIL_CALL_STORE_NAME, - [STORE_SLICE] = _TAIL_CALL_STORE_SLICE, - [STORE_SUBSCR] = _TAIL_CALL_STORE_SUBSCR, - [STORE_SUBSCR_DICT] = _TAIL_CALL_STORE_SUBSCR_DICT, - [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_STORE_SUBSCR_LIST_INT, - [SWAP] = _TAIL_CALL_SWAP, - [TO_BOOL] = _TAIL_CALL_TO_BOOL, - [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TO_BOOL_ALWAYS_TRUE, - [TO_BOOL_BOOL] = _TAIL_CALL_TO_BOOL_BOOL, - [TO_BOOL_INT] = _TAIL_CALL_TO_BOOL_INT, - [TO_BOOL_LIST] = _TAIL_CALL_TO_BOOL_LIST, - [TO_BOOL_NONE] = _TAIL_CALL_TO_BOOL_NONE, - [TO_BOOL_STR] = _TAIL_CALL_TO_BOOL_STR, - [UNARY_INVERT] = _TAIL_CALL_UNARY_INVERT, - [UNARY_NEGATIVE] = _TAIL_CALL_UNARY_NEGATIVE, - [UNARY_NOT] = _TAIL_CALL_UNARY_NOT, - [UNPACK_EX] = _TAIL_CALL_UNPACK_EX, - [UNPACK_SEQUENCE] = _TAIL_CALL_UNPACK_SEQUENCE, - [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_UNPACK_SEQUENCE_LIST, - [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TUPLE, - [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, - [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, - [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, - [118] = _TAIL_CALL_UNKNOWN_OPCODE, - [119] = _TAIL_CALL_UNKNOWN_OPCODE, - [120] = _TAIL_CALL_UNKNOWN_OPCODE, - [121] = _TAIL_CALL_UNKNOWN_OPCODE, - [122] = _TAIL_CALL_UNKNOWN_OPCODE, - [123] = _TAIL_CALL_UNKNOWN_OPCODE, - [124] = _TAIL_CALL_UNKNOWN_OPCODE, - [125] = _TAIL_CALL_UNKNOWN_OPCODE, - [126] = _TAIL_CALL_UNKNOWN_OPCODE, - [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [128] = _TAIL_CALL_UNKNOWN_OPCODE, - [129] = _TAIL_CALL_UNKNOWN_OPCODE, - [130] = _TAIL_CALL_UNKNOWN_OPCODE, - [131] = _TAIL_CALL_UNKNOWN_OPCODE, - [132] = _TAIL_CALL_UNKNOWN_OPCODE, - [133] = _TAIL_CALL_UNKNOWN_OPCODE, - [134] = _TAIL_CALL_UNKNOWN_OPCODE, - [135] = _TAIL_CALL_UNKNOWN_OPCODE, - [136] = _TAIL_CALL_UNKNOWN_OPCODE, - [137] = _TAIL_CALL_UNKNOWN_OPCODE, - [138] = _TAIL_CALL_UNKNOWN_OPCODE, - [139] = _TAIL_CALL_UNKNOWN_OPCODE, - [140] = _TAIL_CALL_UNKNOWN_OPCODE, - [141] = _TAIL_CALL_UNKNOWN_OPCODE, - [142] = _TAIL_CALL_UNKNOWN_OPCODE, - [143] = _TAIL_CALL_UNKNOWN_OPCODE, - [144] = _TAIL_CALL_UNKNOWN_OPCODE, - [145] = _TAIL_CALL_UNKNOWN_OPCODE, - [146] = _TAIL_CALL_UNKNOWN_OPCODE, - [147] = _TAIL_CALL_UNKNOWN_OPCODE, - [148] = _TAIL_CALL_UNKNOWN_OPCODE, - [232] = _TAIL_CALL_UNKNOWN_OPCODE, - [233] = _TAIL_CALL_UNKNOWN_OPCODE, - [234] = _TAIL_CALL_UNKNOWN_OPCODE, -}; -#endif /* Py_TAIL_CALL_INTERP */ #ifndef Py_TAIL_CALL_INTERP #if !USE_COMPUTED_GOTOS @@ -523,10 +19,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { TARGET(BINARY_OP) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = BINARY_OP; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP); @@ -579,14 +73,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(BINARY_OP_ADD_FLOAT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = BINARY_OP_ADD_FLOAT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); @@ -635,14 +125,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(BINARY_OP_ADD_INT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = BINARY_OP_ADD_INT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_INT); @@ -690,14 +176,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(BINARY_OP_ADD_UNICODE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = BINARY_OP_ADD_UNICODE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); @@ -745,10 +227,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(BINARY_OP_EXTEND) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = BINARY_OP_EXTEND; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -801,14 +281,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = BINARY_OP_INPLACE_ADD_UNICODE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); @@ -886,14 +362,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(BINARY_OP_MULTIPLY_FLOAT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = BINARY_OP_MULTIPLY_FLOAT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); @@ -942,14 +414,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(BINARY_OP_MULTIPLY_INT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = BINARY_OP_MULTIPLY_INT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); @@ -997,14 +465,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(BINARY_OP_SUBTRACT_FLOAT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = BINARY_OP_SUBTRACT_FLOAT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); @@ -1053,14 +517,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(BINARY_OP_SUBTRACT_INT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = BINARY_OP_SUBTRACT_INT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); @@ -1108,10 +568,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(BINARY_SLICE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = BINARY_SLICE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BINARY_SLICE); @@ -1168,10 +628,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(BINARY_SUBSCR) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = BINARY_SUBSCR; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR); @@ -1221,14 +679,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(BINARY_SUBSCR_DICT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = BINARY_SUBSCR_DICT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_DICT); @@ -1270,14 +724,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(BINARY_SUBSCR_GETITEM) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = BINARY_SUBSCR_GETITEM; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); @@ -1357,14 +807,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(BINARY_SUBSCR_LIST_INT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = BINARY_SUBSCR_LIST_INT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); @@ -1429,14 +875,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(BINARY_SUBSCR_STR_INT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = BINARY_SUBSCR_STR_INT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); @@ -1493,14 +935,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(BINARY_SUBSCR_TUPLE_INT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = BINARY_SUBSCR_TUPLE_INT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); @@ -1553,10 +991,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(BUILD_LIST) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = BUILD_LIST; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_LIST); @@ -1575,10 +1013,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(BUILD_MAP) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = BUILD_MAP; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_MAP); @@ -1617,10 +1055,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(BUILD_SET) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = BUILD_SET; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SET); @@ -1665,10 +1103,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(BUILD_SLICE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = BUILD_SLICE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SLICE); @@ -1695,10 +1133,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(BUILD_STRING) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = BUILD_STRING; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_STRING); @@ -1732,10 +1170,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(BUILD_TUPLE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = BUILD_TUPLE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_TUPLE); @@ -1754,10 +1192,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CACHE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CACHE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CACHE); @@ -1767,17 +1205,14 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL); PREDICTED_CALL:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; - opcode = CALL; _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; @@ -1937,10 +1372,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_ALLOC_AND_ENTER_INIT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_ALLOC_AND_ENTER_INIT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -2063,10 +1496,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_BOUND_METHOD_EXACT_ARGS; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -2202,10 +1633,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_BOUND_METHOD_GENERAL) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_BOUND_METHOD_GENERAL; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -2324,14 +1753,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_BUILTIN_CLASS) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_BUILTIN_CLASS; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_CLASS); @@ -2418,14 +1843,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_BUILTIN_FAST) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_BUILTIN_FAST; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST); @@ -2518,14 +1939,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); @@ -2619,14 +2036,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_BUILTIN_O) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_BUILTIN_O; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_O); @@ -2717,16 +2130,13 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_FUNCTION_EX) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_FUNCTION_EX; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_FUNCTION_EX); - opcode = CALL_FUNCTION_EX; _PyStackRef func; _PyStackRef callargs; _PyStackRef kwargs_in; @@ -2909,10 +2319,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_INTRINSIC_1) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_INTRINSIC_1; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_1); @@ -2933,10 +2343,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_INTRINSIC_2) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_INTRINSIC_2; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_2); @@ -2964,14 +2374,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_ISINSTANCE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_ISINSTANCE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ISINSTANCE); @@ -3027,17 +2433,14 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_KW) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_KW; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW); PREDICTED_CALL_KW:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; - opcode = CALL_KW; _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; @@ -3196,10 +2599,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_KW_BOUND_METHOD) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_KW_BOUND_METHOD; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -3329,18 +2730,13 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_KW_NON_PY) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_KW_NON_PY; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_NON_PY); - opcode = CALL_KW_NON_PY; static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef *callable; _PyStackRef kwnames; @@ -3444,10 +2840,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_KW_PY) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_KW_PY; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -3551,14 +2945,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_LEN) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_LEN; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LEN); @@ -3620,14 +3010,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_LIST_APPEND) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_LIST_APPEND; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LIST_APPEND); @@ -3686,14 +3072,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_METHOD_DESCRIPTOR_FAST) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_METHOD_DESCRIPTOR_FAST; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); @@ -3792,14 +3174,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); @@ -3898,14 +3276,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_METHOD_DESCRIPTOR_NOARGS; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); @@ -4004,14 +3378,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_METHOD_DESCRIPTOR_O) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_METHOD_DESCRIPTOR_O; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); @@ -4112,18 +3482,13 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_NON_PY_GENERAL) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_NON_PY_GENERAL; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_NON_PY_GENERAL); - opcode = CALL_NON_PY_GENERAL; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; _PyStackRef *self_or_null; @@ -4217,10 +3582,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_PY_EXACT_ARGS) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_PY_EXACT_ARGS; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -4328,10 +3691,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_PY_GENERAL) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_PY_GENERAL; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -4424,14 +3785,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_STR_1) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_STR_1; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_STR_1); @@ -4499,14 +3856,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_TUPLE_1) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_TUPLE_1; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TUPLE_1); @@ -4574,14 +3927,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CALL_TYPE_1) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CALL_TYPE_1; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TYPE_1); @@ -4620,10 +3969,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CHECK_EG_MATCH) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CHECK_EG_MATCH; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EG_MATCH); @@ -4675,10 +4024,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CHECK_EXC_MATCH) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CHECK_EXC_MATCH; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EXC_MATCH); @@ -4707,10 +4056,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CLEANUP_THROW) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CLEANUP_THROW; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -4755,10 +4102,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(COMPARE_OP) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = COMPARE_OP; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP); @@ -4824,14 +4169,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(COMPARE_OP_FLOAT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = COMPARE_OP_FLOAT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_FLOAT); @@ -4878,14 +4219,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(COMPARE_OP_INT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = COMPARE_OP_INT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_INT); @@ -4944,14 +4281,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(COMPARE_OP_STR) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = COMPARE_OP_STR; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_STR); @@ -4999,10 +4332,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CONTAINS_OP) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CONTAINS_OP; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP); @@ -5051,14 +4382,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CONTAINS_OP_DICT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CONTAINS_OP_DICT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_DICT); @@ -5093,14 +4420,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CONTAINS_OP_SET) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CONTAINS_OP_SET; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_SET); @@ -5136,10 +4459,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(CONVERT_VALUE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = CONVERT_VALUE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CONVERT_VALUE); @@ -5168,10 +4491,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(COPY) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = COPY; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY); @@ -5187,10 +4510,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(COPY_FREE_VARS) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = COPY_FREE_VARS; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY_FREE_VARS); @@ -5209,10 +4532,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(DELETE_ATTR) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = DELETE_ATTR; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_ATTR); @@ -5232,10 +4555,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(DELETE_DEREF) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = DELETE_DEREF; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_DEREF); @@ -5256,10 +4579,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(DELETE_FAST) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = DELETE_FAST; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_FAST); @@ -5282,10 +4605,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(DELETE_GLOBAL) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = DELETE_GLOBAL; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_GLOBAL); @@ -5308,10 +4631,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(DELETE_NAME) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = DELETE_NAME; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_NAME); @@ -5341,10 +4664,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(DELETE_SUBSCR) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = DELETE_SUBSCR; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_SUBSCR); @@ -5368,10 +4691,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(DICT_MERGE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = DICT_MERGE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_MERGE); @@ -5401,10 +4724,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(DICT_UPDATE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = DICT_UPDATE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_UPDATE); @@ -5438,10 +4761,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(END_ASYNC_FOR) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = END_ASYNC_FOR; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -5475,10 +4796,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(END_FOR) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = END_FOR; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(END_FOR); _PyStackRef value; @@ -5497,10 +4818,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(END_SEND) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = END_SEND; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_SEND); @@ -5519,16 +4840,13 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(ENTER_EXECUTOR) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = ENTER_EXECUTOR; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(ENTER_EXECUTOR); - opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; @@ -5558,10 +4876,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(EXIT_INIT_CHECK) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = EXIT_INIT_CHECK; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXIT_INIT_CHECK); @@ -5582,14 +4900,13 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(EXTENDED_ARG) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = EXTENDED_ARG; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXTENDED_ARG); - opcode = EXTENDED_ARG; assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; @@ -5598,10 +4915,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(FORMAT_SIMPLE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = FORMAT_SIMPLE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_SIMPLE); @@ -5637,10 +4954,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(FORMAT_WITH_SPEC) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = FORMAT_WITH_SPEC; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_WITH_SPEC); @@ -5665,10 +4982,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(FOR_ITER) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = FOR_ITER; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER); @@ -5731,14 +5046,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(FOR_ITER_GEN) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = FOR_ITER_GEN; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_GEN); @@ -5799,14 +5110,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(FOR_ITER_LIST) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = FOR_ITER_LIST; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_LIST); @@ -5862,14 +5169,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(FOR_ITER_RANGE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = FOR_ITER_RANGE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_RANGE); @@ -5919,14 +5222,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(FOR_ITER_TUPLE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = FOR_ITER_TUPLE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_TUPLE); @@ -5979,10 +5278,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(GET_AITER) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = GET_AITER; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AITER); @@ -6032,10 +5331,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(GET_ANEXT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = GET_ANEXT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ANEXT); @@ -6056,10 +5355,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(GET_AWAITABLE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = GET_AWAITABLE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AWAITABLE); @@ -6079,10 +5378,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(GET_ITER) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = GET_ITER; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ITER); @@ -6103,10 +5402,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(GET_LEN) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = GET_LEN; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_LEN); @@ -6132,10 +5431,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(GET_YIELD_FROM_ITER) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = GET_YIELD_FROM_ITER; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_YIELD_FROM_ITER); @@ -6179,10 +5478,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(IMPORT_FROM) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = IMPORT_FROM; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_FROM); @@ -6204,10 +5503,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(IMPORT_NAME) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = IMPORT_NAME; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_NAME); @@ -6235,16 +5534,13 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(INSTRUMENTED_CALL) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_CALL; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL); - opcode = INSTRUMENTED_CALL; _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; @@ -6416,16 +5712,13 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_CALL_FUNCTION_EX; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - opcode = INSTRUMENTED_CALL_FUNCTION_EX; _PyStackRef func; _PyStackRef callargs; _PyStackRef kwargs_in; @@ -6608,16 +5901,13 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(INSTRUMENTED_CALL_KW) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_CALL_KW; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); - opcode = INSTRUMENTED_CALL_KW; _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; @@ -6786,10 +6076,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(INSTRUMENTED_END_FOR) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_END_FOR; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; next_instr += 1; @@ -6815,10 +6103,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(INSTRUMENTED_END_SEND) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_END_SEND; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -6849,10 +6135,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(INSTRUMENTED_FOR_ITER) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_FOR_ITER; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -6891,16 +6175,13 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(INSTRUMENTED_INSTRUCTION) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_INSTRUCTION; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); - opcode = INSTRUMENTED_INSTRUCTION; _PyFrame_SetStackPointer(frame, stack_pointer); int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); @@ -6918,10 +6199,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_JUMP_BACKWARD; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -6949,10 +6228,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_JUMP_FORWARD; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -6963,17 +6240,14 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(INSTRUMENTED_LINE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_LINE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_LINE); - opcode = INSTRUMENTED_LINE; int original_opcode = 0; if (tstate->tracing) { PyCodeObject *code = _PyFrame_GetCode(frame); @@ -7007,16 +6281,13 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_LOAD_SUPER_ATTR; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); - opcode = INSTRUMENTED_LOAD_SUPER_ATTR; _PyStackRef global_super_st; _PyStackRef class_st; _PyStackRef self_st; @@ -7103,10 +6374,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(INSTRUMENTED_NOT_TAKEN) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_NOT_TAKEN; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -7119,10 +6388,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(INSTRUMENTED_POP_ITER) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_POP_ITER; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -7141,10 +6408,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7162,10 +6427,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_POP_JUMP_IF_NONE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7187,10 +6450,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7210,10 +6471,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7231,10 +6490,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(INSTRUMENTED_RESUME) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_RESUME; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7314,10 +6571,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(INSTRUMENTED_RETURN_VALUE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_RETURN_VALUE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7364,10 +6619,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(INSTRUMENTED_YIELD_VALUE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = INSTRUMENTED_YIELD_VALUE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7436,10 +6689,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(INTERPRETER_EXIT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = INTERPRETER_EXIT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INTERPRETER_EXIT); @@ -7459,10 +6712,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(IS_OP) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = IS_OP; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IS_OP); @@ -7482,10 +6735,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(JUMP_BACKWARD) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = JUMP_BACKWARD; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD); @@ -7531,10 +6782,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(JUMP_BACKWARD_JIT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = JUMP_BACKWARD_JIT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7604,10 +6853,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = JUMP_BACKWARD_NO_INTERRUPT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); @@ -7622,10 +6871,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(JUMP_BACKWARD_NO_JIT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = JUMP_BACKWARD_NO_JIT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); @@ -7658,10 +6907,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(JUMP_FORWARD) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = JUMP_FORWARD; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_FORWARD); @@ -7670,10 +6919,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LIST_APPEND) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LIST_APPEND; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_APPEND); @@ -7692,10 +6941,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LIST_EXTEND) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LIST_EXTEND; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_EXTEND); @@ -7733,10 +6982,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_ATTR) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_ATTR; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR); @@ -7817,10 +7064,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_ATTR_CLASS) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_ATTR_CLASS; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7869,10 +7114,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7931,10 +7174,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7989,10 +7230,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_ATTR_INSTANCE_VALUE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_ATTR_INSTANCE_VALUE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8066,10 +7305,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_ATTR_METHOD_LAZY_DICT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8123,10 +7360,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_ATTR_METHOD_NO_DICT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_ATTR_METHOD_NO_DICT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8169,10 +7404,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_ATTR_METHOD_WITH_VALUES; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8237,10 +7470,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_ATTR_MODULE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_ATTR_MODULE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8316,10 +7547,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8357,10 +7586,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8419,10 +7646,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_ATTR_PROPERTY) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_ATTR_PROPERTY; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8514,10 +7739,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_ATTR_SLOT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_ATTR_SLOT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8577,10 +7800,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_ATTR_WITH_HINT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_ATTR_WITH_HINT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8681,10 +7902,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_BUILD_CLASS) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_BUILD_CLASS; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_BUILD_CLASS); @@ -8711,10 +7932,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_COMMON_CONSTANT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_COMMON_CONSTANT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); @@ -8737,10 +7958,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_CONST) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_CONST; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); @@ -8774,10 +7993,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_CONST_IMMORTAL) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_CONST_IMMORTAL; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST_IMMORTAL); @@ -8793,10 +8012,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_CONST_MORTAL) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_CONST_MORTAL; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST_MORTAL); @@ -8811,10 +8030,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_DEREF) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_DEREF; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_DEREF); @@ -8835,10 +8054,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_FAST) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_FAST; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST); @@ -8852,10 +8071,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_FAST_AND_CLEAR) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_FAST_AND_CLEAR; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); @@ -8869,10 +8088,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_FAST_CHECK) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_FAST_CHECK; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_CHECK); @@ -8895,10 +8114,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_FAST_LOAD_FAST) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_FAST_LOAD_FAST; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); @@ -8916,10 +8135,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_FROM_DICT_OR_DEREF) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_FROM_DICT_OR_DEREF; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); @@ -8961,10 +8180,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_FROM_DICT_OR_GLOBALS) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_FROM_DICT_OR_GLOBALS; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); @@ -9041,10 +8260,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_GLOBAL) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_GLOBAL; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL); @@ -9095,10 +8312,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_GLOBAL_BUILTIN) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_GLOBAL_BUILTIN; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -9179,10 +8394,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_GLOBAL_MODULE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_GLOBAL_MODULE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -9247,10 +8460,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_LOCALS) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_LOCALS; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_LOCALS); @@ -9271,10 +8484,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_NAME) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_NAME; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_NAME); @@ -9294,10 +8507,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_SMALL_INT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_SMALL_INT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SMALL_INT); @@ -9312,10 +8525,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_SPECIAL) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_SPECIAL; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SPECIAL); @@ -9353,17 +8566,14 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_SUPER_ATTR) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_SUPER_ATTR; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR); PREDICTED_LOAD_SUPER_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - opcode = LOAD_SUPER_ATTR; _PyStackRef global_super_st; _PyStackRef class_st; _PyStackRef self_st; @@ -9466,14 +8676,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_SUPER_ATTR_ATTR) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_SUPER_ATTR_ATTR; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); @@ -9519,14 +8725,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(LOAD_SUPER_ATTR_METHOD) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = LOAD_SUPER_ATTR_METHOD; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); @@ -9588,10 +8790,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(MAKE_CELL) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = MAKE_CELL; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_CELL); @@ -9611,10 +8813,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(MAKE_FUNCTION) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = MAKE_FUNCTION; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_FUNCTION); @@ -9644,10 +8846,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(MAP_ADD) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = MAP_ADD; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAP_ADD); @@ -9677,10 +8879,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(MATCH_CLASS) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = MATCH_CLASS; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_CLASS); @@ -9721,10 +8923,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(MATCH_KEYS) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = MATCH_KEYS; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_KEYS); @@ -9749,10 +8951,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(MATCH_MAPPING) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = MATCH_MAPPING; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_MAPPING); @@ -9768,10 +8970,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(MATCH_SEQUENCE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = MATCH_SEQUENCE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_SEQUENCE); @@ -9787,10 +8989,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(NOP) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = NOP; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOP); @@ -9798,10 +9000,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(NOT_TAKEN) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = NOT_TAKEN; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOT_TAKEN); @@ -9809,10 +9011,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(POP_EXCEPT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = POP_EXCEPT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_EXCEPT); @@ -9830,10 +9032,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(POP_ITER) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = POP_ITER; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_ITER); @@ -9846,10 +9048,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(POP_JUMP_IF_FALSE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = POP_JUMP_IF_FALSE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -9868,10 +9068,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(POP_JUMP_IF_NONE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = POP_JUMP_IF_NONE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -9906,10 +9104,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(POP_JUMP_IF_NOT_NONE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = POP_JUMP_IF_NOT_NONE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -9944,10 +9140,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(POP_JUMP_IF_TRUE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = POP_JUMP_IF_TRUE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -9966,10 +9160,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(POP_TOP) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = POP_TOP; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_TOP); @@ -9982,10 +9176,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(PUSH_EXC_INFO) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = PUSH_EXC_INFO; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_EXC_INFO); @@ -10011,10 +9205,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(PUSH_NULL) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = PUSH_NULL; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_NULL); @@ -10027,10 +9221,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(RAISE_VARARGS) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = RAISE_VARARGS; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -10058,10 +9250,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(RERAISE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = RERAISE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -10104,14 +9294,13 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_SetStackPointer(frame, stack_pointer); JUMP_TO_LABEL(exception_unwind); - DISPATCH(); } TARGET(RESERVED) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = RESERVED; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESERVED); @@ -10121,10 +9310,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(RESUME) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = RESUME; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME); @@ -10199,14 +9386,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(RESUME_CHECK) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = RESUME_CHECK; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME_CHECK); @@ -10239,10 +9422,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(RETURN_GENERATOR) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = RETURN_GENERATOR; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); @@ -10278,10 +9461,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(RETURN_VALUE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = RETURN_VALUE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_VALUE); @@ -10310,10 +9493,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(SEND) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = SEND; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND); @@ -10410,14 +9591,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(SEND_GEN) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = SEND_GEN; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND_GEN); @@ -10482,10 +9659,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(SETUP_ANNOTATIONS) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = SETUP_ANNOTATIONS; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SETUP_ANNOTATIONS); @@ -10529,10 +9706,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(SET_ADD) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = SET_ADD; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_ADD); @@ -10554,10 +9731,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(SET_FUNCTION_ATTRIBUTE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = SET_FUNCTION_ATTRIBUTE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); @@ -10582,10 +9759,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(SET_UPDATE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = SET_UPDATE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_UPDATE); @@ -10607,10 +9784,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(STORE_ATTR) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = STORE_ATTR; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR); @@ -10658,10 +9833,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(STORE_ATTR_INSTANCE_VALUE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = STORE_ATTR_INSTANCE_VALUE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -10734,10 +9907,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(STORE_ATTR_SLOT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = STORE_ATTR_SLOT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -10785,10 +9956,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(STORE_ATTR_WITH_HINT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = STORE_ATTR_WITH_HINT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -10885,10 +10054,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(STORE_DEREF) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = STORE_DEREF; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_DEREF); @@ -10904,10 +10073,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(STORE_FAST) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = STORE_FAST; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST); @@ -10924,10 +10093,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(STORE_FAST_LOAD_FAST) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = STORE_FAST_LOAD_FAST; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); @@ -10947,10 +10116,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(STORE_FAST_STORE_FAST) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = STORE_FAST_STORE_FAST; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_STORE_FAST); @@ -10978,10 +10147,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(STORE_GLOBAL) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = STORE_GLOBAL; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_GLOBAL); @@ -11001,10 +10170,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(STORE_NAME) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = STORE_NAME; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_NAME); @@ -11041,10 +10210,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(STORE_SLICE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = STORE_SLICE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_SLICE); @@ -11095,10 +10264,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(STORE_SUBSCR) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = STORE_SUBSCR; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR); @@ -11146,14 +10313,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(STORE_SUBSCR_DICT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = STORE_SUBSCR_DICT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_DICT); @@ -11189,14 +10352,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(STORE_SUBSCR_LIST_INT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = STORE_SUBSCR_LIST_INT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); @@ -11257,10 +10416,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(SWAP) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = SWAP; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SWAP); @@ -11276,10 +10435,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(TO_BOOL) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = TO_BOOL; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL); @@ -11322,10 +10479,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(TO_BOOL_ALWAYS_TRUE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = TO_BOOL_ALWAYS_TRUE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -11359,14 +10514,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(TO_BOOL_BOOL) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = TO_BOOL_BOOL; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_BOOL); @@ -11385,14 +10536,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(TO_BOOL_INT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = TO_BOOL_INT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_INT); @@ -11422,14 +10569,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(TO_BOOL_LIST) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = TO_BOOL_LIST; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_LIST); @@ -11453,14 +10596,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(TO_BOOL_NONE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = TO_BOOL_NONE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_NONE); @@ -11483,14 +10622,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(TO_BOOL_STR) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = TO_BOOL_STR; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_STR); @@ -11521,10 +10656,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(UNARY_INVERT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = UNARY_INVERT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_INVERT); @@ -11544,10 +10679,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(UNARY_NEGATIVE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = UNARY_NEGATIVE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NEGATIVE); @@ -11567,10 +10702,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(UNARY_NOT) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = UNARY_NOT; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NOT); @@ -11585,10 +10720,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(UNPACK_EX) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = UNPACK_EX; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNPACK_EX); @@ -11610,10 +10745,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(UNPACK_SEQUENCE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = UNPACK_SEQUENCE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE); @@ -11659,14 +10792,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(UNPACK_SEQUENCE_LIST) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = UNPACK_SEQUENCE_LIST; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); @@ -11708,14 +10837,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(UNPACK_SEQUENCE_TUPLE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = UNPACK_SEQUENCE_TUPLE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); @@ -11748,14 +10873,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = UNPACK_SEQUENCE_TWO_TUPLE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ - #ifdef Py_TAIL_CALL_INTERP _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; - #endif /* Py_TAIL_CALL_INTERP */ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); @@ -11789,10 +10910,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(WITH_EXCEPT_START) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = WITH_EXCEPT_START; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(WITH_EXCEPT_START); @@ -11847,10 +10968,10 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { } TARGET(YIELD_VALUE) { - #ifdef Py_TAIL_CALL_INTERP - int opcode = next_instr->op.code; + int opcode = YIELD_VALUE; (void)(opcode); - #endif /* Py_TAIL_CALL_INTERP */ + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(YIELD_VALUE); diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 2b84f0281e5356..27c4d537b8079b 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -1,3 +1,4 @@ +#ifndef Py_TAIL_CALL_INTERP static void *opcode_targets[256] = { &&TARGET_CACHE, &&TARGET_BINARY_SLICE, @@ -256,3 +257,507 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_LINE, &&TARGET_ENTER_EXECUTOR, }; +#else /* Py_TAIL_CALL_INTERP */ +static py_tail_call_funcptr INSTRUCTION_TABLE[256]; + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_4_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_3_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_MORTAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); + +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { + int opcode = next_instr->op.code; + _PyErr_Format(tstate, PyExc_SystemError, + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); +JUMP_TO_LABEL(error); +} + +static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { + [BINARY_OP] = _TAIL_CALL_BINARY_OP, + [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, + [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, + [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_BINARY_OP_ADD_UNICODE, + [BINARY_OP_EXTEND] = _TAIL_CALL_BINARY_OP_EXTEND, + [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE, + [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT, + [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_BINARY_OP_MULTIPLY_INT, + [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT, + [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_BINARY_OP_SUBTRACT_INT, + [BINARY_SLICE] = _TAIL_CALL_BINARY_SLICE, + [BINARY_SUBSCR] = _TAIL_CALL_BINARY_SUBSCR, + [BINARY_SUBSCR_DICT] = _TAIL_CALL_BINARY_SUBSCR_DICT, + [BINARY_SUBSCR_GETITEM] = _TAIL_CALL_BINARY_SUBSCR_GETITEM, + [BINARY_SUBSCR_LIST_INT] = _TAIL_CALL_BINARY_SUBSCR_LIST_INT, + [BINARY_SUBSCR_STR_INT] = _TAIL_CALL_BINARY_SUBSCR_STR_INT, + [BINARY_SUBSCR_TUPLE_INT] = _TAIL_CALL_BINARY_SUBSCR_TUPLE_INT, + [BUILD_LIST] = _TAIL_CALL_BUILD_LIST, + [BUILD_MAP] = _TAIL_CALL_BUILD_MAP, + [BUILD_SET] = _TAIL_CALL_BUILD_SET, + [BUILD_SLICE] = _TAIL_CALL_BUILD_SLICE, + [BUILD_STRING] = _TAIL_CALL_BUILD_STRING, + [BUILD_TUPLE] = _TAIL_CALL_BUILD_TUPLE, + [CACHE] = _TAIL_CALL_CACHE, + [CALL] = _TAIL_CALL_CALL, + [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT, + [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS, + [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_CALL_BOUND_METHOD_GENERAL, + [CALL_BUILTIN_CLASS] = _TAIL_CALL_CALL_BUILTIN_CLASS, + [CALL_BUILTIN_FAST] = _TAIL_CALL_CALL_BUILTIN_FAST, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS, + [CALL_BUILTIN_O] = _TAIL_CALL_CALL_BUILTIN_O, + [CALL_FUNCTION_EX] = _TAIL_CALL_CALL_FUNCTION_EX, + [CALL_INTRINSIC_1] = _TAIL_CALL_CALL_INTRINSIC_1, + [CALL_INTRINSIC_2] = _TAIL_CALL_CALL_INTRINSIC_2, + [CALL_ISINSTANCE] = _TAIL_CALL_CALL_ISINSTANCE, + [CALL_KW] = _TAIL_CALL_CALL_KW, + [CALL_KW_BOUND_METHOD] = _TAIL_CALL_CALL_KW_BOUND_METHOD, + [CALL_KW_NON_PY] = _TAIL_CALL_CALL_KW_NON_PY, + [CALL_KW_PY] = _TAIL_CALL_CALL_KW_PY, + [CALL_LEN] = _TAIL_CALL_CALL_LEN, + [CALL_LIST_APPEND] = _TAIL_CALL_CALL_LIST_APPEND, + [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS, + [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O, + [CALL_NON_PY_GENERAL] = _TAIL_CALL_CALL_NON_PY_GENERAL, + [CALL_PY_EXACT_ARGS] = _TAIL_CALL_CALL_PY_EXACT_ARGS, + [CALL_PY_GENERAL] = _TAIL_CALL_CALL_PY_GENERAL, + [CALL_STR_1] = _TAIL_CALL_CALL_STR_1, + [CALL_TUPLE_1] = _TAIL_CALL_CALL_TUPLE_1, + [CALL_TYPE_1] = _TAIL_CALL_CALL_TYPE_1, + [CHECK_EG_MATCH] = _TAIL_CALL_CHECK_EG_MATCH, + [CHECK_EXC_MATCH] = _TAIL_CALL_CHECK_EXC_MATCH, + [CLEANUP_THROW] = _TAIL_CALL_CLEANUP_THROW, + [COMPARE_OP] = _TAIL_CALL_COMPARE_OP, + [COMPARE_OP_FLOAT] = _TAIL_CALL_COMPARE_OP_FLOAT, + [COMPARE_OP_INT] = _TAIL_CALL_COMPARE_OP_INT, + [COMPARE_OP_STR] = _TAIL_CALL_COMPARE_OP_STR, + [CONTAINS_OP] = _TAIL_CALL_CONTAINS_OP, + [CONTAINS_OP_DICT] = _TAIL_CALL_CONTAINS_OP_DICT, + [CONTAINS_OP_SET] = _TAIL_CALL_CONTAINS_OP_SET, + [CONVERT_VALUE] = _TAIL_CALL_CONVERT_VALUE, + [COPY] = _TAIL_CALL_COPY, + [COPY_FREE_VARS] = _TAIL_CALL_COPY_FREE_VARS, + [DELETE_ATTR] = _TAIL_CALL_DELETE_ATTR, + [DELETE_DEREF] = _TAIL_CALL_DELETE_DEREF, + [DELETE_FAST] = _TAIL_CALL_DELETE_FAST, + [DELETE_GLOBAL] = _TAIL_CALL_DELETE_GLOBAL, + [DELETE_NAME] = _TAIL_CALL_DELETE_NAME, + [DELETE_SUBSCR] = _TAIL_CALL_DELETE_SUBSCR, + [DICT_MERGE] = _TAIL_CALL_DICT_MERGE, + [DICT_UPDATE] = _TAIL_CALL_DICT_UPDATE, + [END_ASYNC_FOR] = _TAIL_CALL_END_ASYNC_FOR, + [END_FOR] = _TAIL_CALL_END_FOR, + [END_SEND] = _TAIL_CALL_END_SEND, + [ENTER_EXECUTOR] = _TAIL_CALL_ENTER_EXECUTOR, + [EXIT_INIT_CHECK] = _TAIL_CALL_EXIT_INIT_CHECK, + [EXTENDED_ARG] = _TAIL_CALL_EXTENDED_ARG, + [FORMAT_SIMPLE] = _TAIL_CALL_FORMAT_SIMPLE, + [FORMAT_WITH_SPEC] = _TAIL_CALL_FORMAT_WITH_SPEC, + [FOR_ITER] = _TAIL_CALL_FOR_ITER, + [FOR_ITER_GEN] = _TAIL_CALL_FOR_ITER_GEN, + [FOR_ITER_LIST] = _TAIL_CALL_FOR_ITER_LIST, + [FOR_ITER_RANGE] = _TAIL_CALL_FOR_ITER_RANGE, + [FOR_ITER_TUPLE] = _TAIL_CALL_FOR_ITER_TUPLE, + [GET_AITER] = _TAIL_CALL_GET_AITER, + [GET_ANEXT] = _TAIL_CALL_GET_ANEXT, + [GET_AWAITABLE] = _TAIL_CALL_GET_AWAITABLE, + [GET_ITER] = _TAIL_CALL_GET_ITER, + [GET_LEN] = _TAIL_CALL_GET_LEN, + [GET_YIELD_FROM_ITER] = _TAIL_CALL_GET_YIELD_FROM_ITER, + [IMPORT_FROM] = _TAIL_CALL_IMPORT_FROM, + [IMPORT_NAME] = _TAIL_CALL_IMPORT_NAME, + [INSTRUMENTED_CALL] = _TAIL_CALL_INSTRUMENTED_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_CALL_KW] = _TAIL_CALL_INSTRUMENTED_CALL_KW, + [INSTRUMENTED_END_FOR] = _TAIL_CALL_INSTRUMENTED_END_FOR, + [INSTRUMENTED_END_SEND] = _TAIL_CALL_INSTRUMENTED_END_SEND, + [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_INSTRUMENTED_FOR_ITER, + [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_INSTRUMENTED_INSTRUCTION, + [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD, + [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD, + [INSTRUMENTED_LINE] = _TAIL_CALL_INSTRUMENTED_LINE, + [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR, + [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_INSTRUMENTED_NOT_TAKEN, + [INSTRUMENTED_POP_ITER] = _TAIL_CALL_INSTRUMENTED_POP_ITER, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE, + [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE, + [INSTRUMENTED_RESUME] = _TAIL_CALL_INSTRUMENTED_RESUME, + [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_INSTRUMENTED_RETURN_VALUE, + [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_INSTRUMENTED_YIELD_VALUE, + [INTERPRETER_EXIT] = _TAIL_CALL_INTERPRETER_EXIT, + [IS_OP] = _TAIL_CALL_IS_OP, + [JUMP_BACKWARD] = _TAIL_CALL_JUMP_BACKWARD, + [JUMP_BACKWARD_JIT] = _TAIL_CALL_JUMP_BACKWARD_JIT, + [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT, + [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_JUMP_BACKWARD_NO_JIT, + [JUMP_FORWARD] = _TAIL_CALL_JUMP_FORWARD, + [LIST_APPEND] = _TAIL_CALL_LIST_APPEND, + [LIST_EXTEND] = _TAIL_CALL_LIST_EXTEND, + [LOAD_ATTR] = _TAIL_CALL_LOAD_ATTR, + [LOAD_ATTR_CLASS] = _TAIL_CALL_LOAD_ATTR_CLASS, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, + [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE, + [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT, + [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT, + [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES, + [LOAD_ATTR_MODULE] = _TAIL_CALL_LOAD_ATTR_MODULE, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, + [LOAD_ATTR_PROPERTY] = _TAIL_CALL_LOAD_ATTR_PROPERTY, + [LOAD_ATTR_SLOT] = _TAIL_CALL_LOAD_ATTR_SLOT, + [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_LOAD_ATTR_WITH_HINT, + [LOAD_BUILD_CLASS] = _TAIL_CALL_LOAD_BUILD_CLASS, + [LOAD_COMMON_CONSTANT] = _TAIL_CALL_LOAD_COMMON_CONSTANT, + [LOAD_CONST] = _TAIL_CALL_LOAD_CONST, + [LOAD_CONST_IMMORTAL] = _TAIL_CALL_LOAD_CONST_IMMORTAL, + [LOAD_CONST_MORTAL] = _TAIL_CALL_LOAD_CONST_MORTAL, + [LOAD_DEREF] = _TAIL_CALL_LOAD_DEREF, + [LOAD_FAST] = _TAIL_CALL_LOAD_FAST, + [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_LOAD_FAST_AND_CLEAR, + [LOAD_FAST_CHECK] = _TAIL_CALL_LOAD_FAST_CHECK, + [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_LOAD_FAST_LOAD_FAST, + [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF, + [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS, + [LOAD_GLOBAL] = _TAIL_CALL_LOAD_GLOBAL, + [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_LOAD_GLOBAL_BUILTIN, + [LOAD_GLOBAL_MODULE] = _TAIL_CALL_LOAD_GLOBAL_MODULE, + [LOAD_LOCALS] = _TAIL_CALL_LOAD_LOCALS, + [LOAD_NAME] = _TAIL_CALL_LOAD_NAME, + [LOAD_SMALL_INT] = _TAIL_CALL_LOAD_SMALL_INT, + [LOAD_SPECIAL] = _TAIL_CALL_LOAD_SPECIAL, + [LOAD_SUPER_ATTR] = _TAIL_CALL_LOAD_SUPER_ATTR, + [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_LOAD_SUPER_ATTR_ATTR, + [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_LOAD_SUPER_ATTR_METHOD, + [MAKE_CELL] = _TAIL_CALL_MAKE_CELL, + [MAKE_FUNCTION] = _TAIL_CALL_MAKE_FUNCTION, + [MAP_ADD] = _TAIL_CALL_MAP_ADD, + [MATCH_CLASS] = _TAIL_CALL_MATCH_CLASS, + [MATCH_KEYS] = _TAIL_CALL_MATCH_KEYS, + [MATCH_MAPPING] = _TAIL_CALL_MATCH_MAPPING, + [MATCH_SEQUENCE] = _TAIL_CALL_MATCH_SEQUENCE, + [NOP] = _TAIL_CALL_NOP, + [NOT_TAKEN] = _TAIL_CALL_NOT_TAKEN, + [POP_EXCEPT] = _TAIL_CALL_POP_EXCEPT, + [POP_ITER] = _TAIL_CALL_POP_ITER, + [POP_JUMP_IF_FALSE] = _TAIL_CALL_POP_JUMP_IF_FALSE, + [POP_JUMP_IF_NONE] = _TAIL_CALL_POP_JUMP_IF_NONE, + [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_POP_JUMP_IF_NOT_NONE, + [POP_JUMP_IF_TRUE] = _TAIL_CALL_POP_JUMP_IF_TRUE, + [POP_TOP] = _TAIL_CALL_POP_TOP, + [PUSH_EXC_INFO] = _TAIL_CALL_PUSH_EXC_INFO, + [PUSH_NULL] = _TAIL_CALL_PUSH_NULL, + [RAISE_VARARGS] = _TAIL_CALL_RAISE_VARARGS, + [RERAISE] = _TAIL_CALL_RERAISE, + [RESERVED] = _TAIL_CALL_RESERVED, + [RESUME] = _TAIL_CALL_RESUME, + [RESUME_CHECK] = _TAIL_CALL_RESUME_CHECK, + [RETURN_GENERATOR] = _TAIL_CALL_RETURN_GENERATOR, + [RETURN_VALUE] = _TAIL_CALL_RETURN_VALUE, + [SEND] = _TAIL_CALL_SEND, + [SEND_GEN] = _TAIL_CALL_SEND_GEN, + [SETUP_ANNOTATIONS] = _TAIL_CALL_SETUP_ANNOTATIONS, + [SET_ADD] = _TAIL_CALL_SET_ADD, + [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_SET_FUNCTION_ATTRIBUTE, + [SET_UPDATE] = _TAIL_CALL_SET_UPDATE, + [STORE_ATTR] = _TAIL_CALL_STORE_ATTR, + [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE, + [STORE_ATTR_SLOT] = _TAIL_CALL_STORE_ATTR_SLOT, + [STORE_ATTR_WITH_HINT] = _TAIL_CALL_STORE_ATTR_WITH_HINT, + [STORE_DEREF] = _TAIL_CALL_STORE_DEREF, + [STORE_FAST] = _TAIL_CALL_STORE_FAST, + [STORE_FAST_LOAD_FAST] = _TAIL_CALL_STORE_FAST_LOAD_FAST, + [STORE_FAST_STORE_FAST] = _TAIL_CALL_STORE_FAST_STORE_FAST, + [STORE_GLOBAL] = _TAIL_CALL_STORE_GLOBAL, + [STORE_NAME] = _TAIL_CALL_STORE_NAME, + [STORE_SLICE] = _TAIL_CALL_STORE_SLICE, + [STORE_SUBSCR] = _TAIL_CALL_STORE_SUBSCR, + [STORE_SUBSCR_DICT] = _TAIL_CALL_STORE_SUBSCR_DICT, + [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_STORE_SUBSCR_LIST_INT, + [SWAP] = _TAIL_CALL_SWAP, + [TO_BOOL] = _TAIL_CALL_TO_BOOL, + [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TO_BOOL_ALWAYS_TRUE, + [TO_BOOL_BOOL] = _TAIL_CALL_TO_BOOL_BOOL, + [TO_BOOL_INT] = _TAIL_CALL_TO_BOOL_INT, + [TO_BOOL_LIST] = _TAIL_CALL_TO_BOOL_LIST, + [TO_BOOL_NONE] = _TAIL_CALL_TO_BOOL_NONE, + [TO_BOOL_STR] = _TAIL_CALL_TO_BOOL_STR, + [UNARY_INVERT] = _TAIL_CALL_UNARY_INVERT, + [UNARY_NEGATIVE] = _TAIL_CALL_UNARY_NEGATIVE, + [UNARY_NOT] = _TAIL_CALL_UNARY_NOT, + [UNPACK_EX] = _TAIL_CALL_UNPACK_EX, + [UNPACK_SEQUENCE] = _TAIL_CALL_UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_UNPACK_SEQUENCE_LIST, + [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TUPLE, + [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, + [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, + [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, + [118] = _TAIL_CALL_UNKNOWN_OPCODE, + [119] = _TAIL_CALL_UNKNOWN_OPCODE, + [120] = _TAIL_CALL_UNKNOWN_OPCODE, + [121] = _TAIL_CALL_UNKNOWN_OPCODE, + [122] = _TAIL_CALL_UNKNOWN_OPCODE, + [123] = _TAIL_CALL_UNKNOWN_OPCODE, + [124] = _TAIL_CALL_UNKNOWN_OPCODE, + [125] = _TAIL_CALL_UNKNOWN_OPCODE, + [126] = _TAIL_CALL_UNKNOWN_OPCODE, + [127] = _TAIL_CALL_UNKNOWN_OPCODE, + [128] = _TAIL_CALL_UNKNOWN_OPCODE, + [129] = _TAIL_CALL_UNKNOWN_OPCODE, + [130] = _TAIL_CALL_UNKNOWN_OPCODE, + [131] = _TAIL_CALL_UNKNOWN_OPCODE, + [132] = _TAIL_CALL_UNKNOWN_OPCODE, + [133] = _TAIL_CALL_UNKNOWN_OPCODE, + [134] = _TAIL_CALL_UNKNOWN_OPCODE, + [135] = _TAIL_CALL_UNKNOWN_OPCODE, + [136] = _TAIL_CALL_UNKNOWN_OPCODE, + [137] = _TAIL_CALL_UNKNOWN_OPCODE, + [138] = _TAIL_CALL_UNKNOWN_OPCODE, + [139] = _TAIL_CALL_UNKNOWN_OPCODE, + [140] = _TAIL_CALL_UNKNOWN_OPCODE, + [141] = _TAIL_CALL_UNKNOWN_OPCODE, + [142] = _TAIL_CALL_UNKNOWN_OPCODE, + [143] = _TAIL_CALL_UNKNOWN_OPCODE, + [144] = _TAIL_CALL_UNKNOWN_OPCODE, + [145] = _TAIL_CALL_UNKNOWN_OPCODE, + [146] = _TAIL_CALL_UNKNOWN_OPCODE, + [147] = _TAIL_CALL_UNKNOWN_OPCODE, + [148] = _TAIL_CALL_UNKNOWN_OPCODE, + [232] = _TAIL_CALL_UNKNOWN_OPCODE, + [233] = _TAIL_CALL_UNKNOWN_OPCODE, + [234] = _TAIL_CALL_UNKNOWN_OPCODE, +}; +#endif /* Py_TAIL_CALL_INTERP */ diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 30ab0e0385b0bd..cd52f181b60e43 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -1,10 +1,8 @@ from pathlib import Path -from typing import TextIO from analyzer import ( Instruction, Uop, - Label, Properties, StackItem, analysis_error, @@ -14,7 +12,7 @@ from cwriter import CWriter from typing import Callable, TextIO, Iterator, Iterable from lexer import Token -from stack import Storage, StackError, Stack +from stack import Storage, StackError # Set this to true for voluminous output showing state of stack and locals PRINT_STACKS = False @@ -404,7 +402,7 @@ def stack_pointer( self.emit(tkn) return True - def goto_label(self, goto: Token, label: Token, storage: Storage, wrap_paren: bool = False) -> None: + def goto_label(self, goto: Token, label: Token, storage: Storage) -> None: if label.text not in self.labels: print(self.labels.keys()) raise analysis_error(f"Label '{label.text}' does not exist", label) @@ -414,12 +412,10 @@ def goto_label(self, goto: Token, label: Token, storage: Storage, wrap_paren: bo self.emit_save(storage) elif storage.spilled: raise analysis_error("Cannot jump from spilled label without reloading the stack pointer", goto) - self.out.emit(goto) - if wrap_paren: - self.out.emit("(") + self.out.start_line() + self.out.emit("JUMP_TO_LABEL(") self.out.emit(label) - if wrap_paren: - self.out.emit(")") + self.out.emit(")") def emit_save(self, storage: Storage) -> None: storage.save(self.out) @@ -626,14 +622,7 @@ def _emit_block( if tkn.text.startswith("DISPATCH"): self._print_storage(storage) reachable = False - if tkn.text.startswith("JUMP_TO_LABEL"): - next(tkn_iter) - label_tkn = next(tkn_iter) - next(tkn_iter) - self.goto_label(tkn, label_tkn, storage, wrap_paren=True) - reachable = False - else: - self.out.emit(tkn) + self.out.emit(tkn) elif tkn.kind == "IF": self.out.emit(tkn) if_reachable, rbrace, storage = self._emit_if(tkn_iter, uop, storage, inst) @@ -664,20 +653,6 @@ def emit_tokens( raise analysis_error(ex.args[0], rbrace) from None return storage - def emit_label( - self, - label: Label - ) -> None: - tkn_iter = TokenIterator(label.body) - self.out.start_line() - for tkn in tkn_iter: - if tkn.text in self._replacers: - storage = Storage(Stack(), [],[], []) - self._replacers[tkn.text](tkn, tkn_iter, label, storage, None) - continue - self.out.emit(tkn) - - def emit(self, txt: str | Token) -> None: self.out.emit(txt) diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 3f162e72aa7303..6c33debd58e1fe 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -112,7 +112,7 @@ def emit_save(self, storage: Storage) -> None: def emit_reload(self, storage: Storage) -> None: pass - def goto_label(self, goto: Token, label: Token, storage: Storage, wrap_paren: bool = False) -> None: + def goto_label(self, goto: Token, label: Token, storage: Storage) -> None: self.out.emit(goto) self.out.emit(label) diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index c5097b7584724c..db028116db93d2 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -13,6 +13,7 @@ DEFAULT_INPUT, ROOT, ) +from tier1_generator import UNKNOWN_OPCODE_HANDLER from cwriter import CWriter @@ -25,11 +26,49 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: for name, op in analysis.opmap.items(): if op < 256: targets[op] = f"&&TARGET_{name},\n" + out.emit("#ifndef Py_TAIL_CALL_INTERP\n") out.emit("static void *opcode_targets[256] = {\n") for target in targets: out.emit(target) out.emit("};\n") + out.emit("#else /* Py_TAIL_CALL_INTERP */\n") +def function_proto(name: str) -> str: + return f"Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_{name}(TAIL_CALL_PARAMS)" + + +def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: + out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256];\n") + out.emit("\n") + + # Emit function prototypes for labels. + for name in analysis.labels: + out.emit(f"{function_proto(name)};\n") + out.emit("\n") + + # Emit function prototypes for opcode handlers. + for name in sorted(analysis.instructions.keys()): + out.emit(f"{function_proto(name)};\n") + out.emit("\n") + + # Emit unknown opcode handler. + out.emit(function_proto("UNKNOWN_OPCODE")) + out.emit(" {\n") + out.emit("int opcode = next_instr->op.code;\n") + out.emit(UNKNOWN_OPCODE_HANDLER) + out.emit("}\n") + out.emit("\n") + + # Emit the dispatch table. + out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256] = {\n") + for name in sorted(analysis.instructions.keys()): + out.emit(f"[{name}] = _TAIL_CALL_{name},\n") + named_values = analysis.opmap.values() + for rest in range(256): + if rest not in named_values: + out.emit(f"[{rest}] = _TAIL_CALL_UNKNOWN_OPCODE,\n") + out.emit("};\n") + outfile.write("#endif /* Py_TAIL_CALL_INTERP */\n") arg_parser = argparse.ArgumentParser( description="Generate the file with dispatch targets.", @@ -52,3 +91,4 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: with open(args.output, "w") as outfile: out = CWriter(outfile, 0, False) write_opcode_targets(data, out) + write_tailcall_dispatch_table(data, out) diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 8c1cb8a6a4ee4f..3aeb25546739c8 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -119,7 +119,7 @@ def write_uop( raise analysis_error(ex.args[0], uop.body[0]) -def uses_this(inst: Instruction) -> tuple[bool, bool]: +def uses_this(inst: Instruction) -> bool: if inst.properties.needs_this: return True, False for uop in inst.parts: @@ -150,7 +150,7 @@ def uses_this(inst: Instruction) -> tuple[bool, bool]: """ def generate_tier1( - filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool, tail_call_mode: bool + filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool ) -> None: write_header(__file__, filenames, outfile) outfile.write(""" @@ -159,7 +159,6 @@ def generate_tier1( #endif #define TIER_ONE 1 """) - generate_tier1_tailcall_metadata(analysis, outfile, lines) outfile.write(f""" #ifndef Py_TAIL_CALL_INTERP #if !USE_COMPUTED_GOTOS @@ -171,7 +170,7 @@ def generate_tier1( {INSTRUCTION_START_MARKER} """ ) - generate_tier1_cases(analysis, outfile, lines, tail_call_mode) + generate_tier1_cases(analysis, outfile, lines) outfile.write(f""" {INSTRUCTION_END_MARKER} #ifndef Py_TAIL_CALL_INTERP @@ -219,51 +218,8 @@ def generate_tier1_labels( emitter.emit("\n") - -def function_proto(name: str) -> str: - return f"Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_{name}(TAIL_CALL_PARAMS)" - - -def generate_tier1_tailcall_metadata( - analysis: Analysis, outfile: TextIO, lines: bool -) -> None: - - out = CWriter(outfile, 0, lines) - out.emit("#ifdef Py_TAIL_CALL_INTERP\n") - out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256];\n") - out.emit("\n") - - # Emit function prototypes for labels. - for name in analysis.labels: - out.emit(f"{function_proto(name)};\n") - out.emit("\n") - - # Emit function prototypes for opcode handlers. - for name in sorted(analysis.instructions.keys()): - out.emit(f"{function_proto(name)};\n") - out.emit("\n") - - # Emit unknown opcode handler. - out.emit(function_proto("UNKNOWN_OPCODE")) - out.emit(" {\n") - out.emit("int opcode = next_instr->op.code;\n") - out.emit(UNKNOWN_OPCODE_HANDLER) - out.emit("}\n") - out.emit("\n") - - # Emit the dispatch table. - out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256] = {\n") - for name in sorted(analysis.instructions.keys()): - out.emit(f"[{name}] = _TAIL_CALL_{name},\n") - named_values = analysis.opmap.values() - for rest in range(256): - if rest not in named_values: - out.emit(f"[{rest}] = _TAIL_CALL_UNKNOWN_OPCODE,\n") - out.emit("};\n") - outfile.write("#endif /* Py_TAIL_CALL_INTERP */\n") - def generate_tier1_cases( - analysis: Analysis, outfile: TextIO, lines: bool, tail_call_mode: bool + analysis: Analysis, outfile: TextIO, lines: bool ) -> None: out = CWriter(outfile, 2, lines) emitter = Emitter(out, analysis.labels) @@ -271,26 +227,16 @@ def generate_tier1_cases( for name, inst in sorted(analysis.instructions.items()): out.emit("\n") out.emit(f"TARGET({name}) {{\n") - if tail_call_mode: - out.emit(f"#ifdef Py_TAIL_CALL_INTERP\n") - out.emit(f"int opcode = next_instr->op.code;\n") - out.emit(f"(void)(opcode);\n") - out.emit(f"#endif /* Py_TAIL_CALL_INTERP */\n") - needs_this, only_for_tail_call = uses_this(inst) - if not tail_call_mode: - if only_for_tail_call and needs_this: - only_for_tail_call = needs_this = False + out.emit(f"int opcode = {name};\n") + out.emit(f"(void)(opcode);\n") + needs_this = uses_this(inst) unused_guard = "(void)this_instr;\n" if inst.properties.needs_prev: out.emit(f"_Py_CODEUNIT* const prev_instr = frame->instr_ptr;\n") if needs_this and not inst.is_target: - if only_for_tail_call: - out.emit("#ifdef Py_TAIL_CALL_INTERP\n") out.emit(f"_Py_CODEUNIT* const this_instr = next_instr;\n") out.emit(unused_guard) - if only_for_tail_call: - out.emit("#endif /* Py_TAIL_CALL_INTERP */\n") if not inst.properties.no_save_ip: out.emit(f"frame->instr_ptr = next_instr;\n") @@ -301,8 +247,6 @@ def generate_tier1_cases( if needs_this: out.emit(f"_Py_CODEUNIT* const this_instr = next_instr - {inst.size};\n") out.emit(unused_guard) - if inst.properties.uses_opcode: - out.emit(f"opcode = {name};\n") if inst.family is not None: out.emit( f"static_assert({inst.family.size} == {inst.size-1}" @@ -344,11 +288,11 @@ def generate_tier1_cases( def generate_tier1_from_files( - filenames: list[str], outfilename: str, lines: bool, tail_call_mode: bool = False + filenames: list[str], outfilename: str, lines: bool ) -> None: data = analyze_files(filenames) with open(outfilename, "w") as outfile: - generate_tier1(filenames, data, outfile, lines, tail_call_mode) + generate_tier1(filenames, data, outfile, lines) if __name__ == "__main__": @@ -357,4 +301,4 @@ def generate_tier1_from_files( args.input.append(DEFAULT_INPUT) data = analyze_files(args.input) with open(args.output, "w") as outfile: - generate_tier1(args.input, data, outfile, args.emit_line_directives, True) + generate_tier1(args.input, data, outfile, args.emit_line_directives) diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index bfc71676f739ae..5e23360cdc0aaf 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -14,7 +14,6 @@ analyze_files, StackItem, analysis_error, - Label, ) from generators_common import ( DEFAULT_INPUT, From d3f3b97b7e7f3dd4ec7bee6b0e82e7c018040c63 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 6 Feb 2025 22:03:55 +0800 Subject: [PATCH 108/110] Update tier1_generator.py --- Tools/cases_generator/tier1_generator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 3aeb25546739c8..503516f95f046c 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -121,13 +121,13 @@ def write_uop( def uses_this(inst: Instruction) -> bool: if inst.properties.needs_this: - return True, False + return True for uop in inst.parts: if not isinstance(uop, Uop): continue for cache in uop.caches: if cache.name != "unused": - return True, False + return True # Can't be merged into the loop above, because # this must strictly be performed at the end. for uop in inst.parts: @@ -136,7 +136,7 @@ def uses_this(inst: Instruction) -> bool: for tkn in uop.body: if (tkn.kind == "IDENTIFIER" and (tkn.text in {"DEOPT_IF", "EXIT_IF"})): - return True, True + return True return False, False From bac68a30e75258bd98d5eb8420bc669c85a98fc0 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 6 Feb 2025 22:21:29 +0800 Subject: [PATCH 109/110] Fix generated cases --- Lib/test/test_generated_cases.py | 94 ------------ Python/generated_cases.c.h | 188 ----------------------- Tools/cases_generator/tier1_generator.py | 2 +- 3 files changed, 1 insertion(+), 283 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index aa0899ecaf860f..e426311c1560b5 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -306,8 +306,6 @@ def test_inst_no_args(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -328,8 +326,6 @@ def test_inst_one_pop(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -353,8 +349,6 @@ def test_inst_one_push(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -379,8 +373,6 @@ def test_inst_one_push_one_pop(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -406,8 +398,6 @@ def test_binary_op(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -436,8 +426,6 @@ def test_overlap(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -472,8 +460,6 @@ def test_predictions(self): next_instr += 1; INSTRUCTION_STATS(OP1); PREDICTED_OP1:; - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; _PyStackRef res; res = Py_None; stack_pointer[-1] = res; @@ -519,8 +505,6 @@ def test_sync_sp(self): TARGET(A) { int opcode = A; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(A); @@ -540,8 +524,6 @@ def test_sync_sp(self): TARGET(B) { int opcode = B; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(B); @@ -581,8 +563,6 @@ def test_error_if_plain(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -604,8 +584,6 @@ def test_error_if_plain_with_comment(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -631,8 +609,6 @@ def test_error_if_pop(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -666,8 +642,6 @@ def test_error_if_pop_with_result(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -726,8 +700,6 @@ def test_suppress_dispatch(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -820,8 +792,6 @@ def test_macro_instruction(self): TARGET(OP3) { int opcode = OP3; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(OP3); @@ -855,8 +825,6 @@ def test_unused_caches(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(OP); @@ -881,8 +849,6 @@ def test_pseudo_instruction_no_flags(self): TARGET(OP1) { int opcode = OP1; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP1); @@ -904,8 +870,6 @@ def test_pseudo_instruction_with_flags(self): TARGET(OP1) { int opcode = OP1; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP1); @@ -930,8 +894,6 @@ def test_pseudo_instruction_as_sequence(self): TARGET(OP1) { int opcode = OP1; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP1); @@ -941,8 +903,6 @@ def test_pseudo_instruction_as_sequence(self): TARGET(OP2) { int opcode = OP2; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP2); @@ -962,8 +922,6 @@ def test_array_input(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -989,8 +947,6 @@ def test_array_output(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1021,8 +977,6 @@ def test_array_input_output(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1049,8 +1003,6 @@ def test_array_error_if(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1079,8 +1031,6 @@ def test_cond_effect(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1124,8 +1074,6 @@ def test_macro_cond_effect(self): TARGET(M) { int opcode = M; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); @@ -1172,8 +1120,6 @@ def test_macro_push_push(self): TARGET(M) { int opcode = M; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); @@ -1209,8 +1155,6 @@ def test_override_inst(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1234,8 +1178,6 @@ def test_override_op(self): TARGET(M) { int opcode = M; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); @@ -1255,8 +1197,6 @@ def test_annotated_inst(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1277,8 +1217,6 @@ def test_annotated_op(self): TARGET(M) { int opcode = M; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); @@ -1317,8 +1255,6 @@ def test_array_of_one(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1343,8 +1279,6 @@ def test_pointer_to_stackref(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1385,8 +1319,6 @@ def test_unused_named_values(self): TARGET(INST) { int opcode = INST; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INST); @@ -1415,8 +1347,6 @@ def test_used_unused_used(self): TARGET(TEST) { int opcode = TEST; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(TEST); @@ -1459,8 +1389,6 @@ def test_unused_used_used(self): TARGET(TEST) { int opcode = TEST; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(TEST); @@ -1502,8 +1430,6 @@ def test_flush(self): TARGET(TEST) { int opcode = TEST; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(TEST); @@ -1554,8 +1480,6 @@ def test_pop_on_error_peeks(self): TARGET(TEST) { int opcode = TEST; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(TEST); @@ -1606,8 +1530,6 @@ def test_push_then_error(self): TARGET(TEST) { int opcode = TEST; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(TEST); @@ -1651,8 +1573,6 @@ def test_error_if_true(self): TARGET(OP1) { int opcode = OP1; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP1); @@ -1662,8 +1582,6 @@ def test_error_if_true(self): TARGET(OP2) { int opcode = OP2; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP2); @@ -1723,8 +1641,6 @@ def test_stack_save_reload(self): TARGET(BALANCED) { int opcode = BALANCED; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BALANCED); @@ -1747,8 +1663,6 @@ def test_stack_reload_only(self): TARGET(BALANCED) { int opcode = BALANCED; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BALANCED); @@ -1796,8 +1710,6 @@ def test_instruction_size_macro(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1835,8 +1747,6 @@ def test_escaping_call_next_to_cmacro(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1873,8 +1783,6 @@ def test_pystackref_frompyobject_new_next_to_cmacro(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); @@ -1908,8 +1816,6 @@ def test_pop_input(self): TARGET(OP) { int opcode = OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 828954efe065ef..69a77e7cd71e34 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -570,8 +570,6 @@ TARGET(BINARY_SLICE) { int opcode = BINARY_SLICE; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BINARY_SLICE); @@ -993,8 +991,6 @@ TARGET(BUILD_LIST) { int opcode = BUILD_LIST; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_LIST); @@ -1015,8 +1011,6 @@ TARGET(BUILD_MAP) { int opcode = BUILD_MAP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_MAP); @@ -1057,8 +1051,6 @@ TARGET(BUILD_SET) { int opcode = BUILD_SET; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SET); @@ -1105,8 +1097,6 @@ TARGET(BUILD_SLICE) { int opcode = BUILD_SLICE; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SLICE); @@ -1135,8 +1125,6 @@ TARGET(BUILD_STRING) { int opcode = BUILD_STRING; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_STRING); @@ -1172,8 +1160,6 @@ TARGET(BUILD_TUPLE) { int opcode = BUILD_TUPLE; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_TUPLE); @@ -1194,8 +1180,6 @@ TARGET(CACHE) { int opcode = CACHE; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CACHE); @@ -2321,8 +2305,6 @@ TARGET(CALL_INTRINSIC_1) { int opcode = CALL_INTRINSIC_1; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_1); @@ -2345,8 +2327,6 @@ TARGET(CALL_INTRINSIC_2) { int opcode = CALL_INTRINSIC_2; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_2); @@ -3971,8 +3951,6 @@ TARGET(CHECK_EG_MATCH) { int opcode = CHECK_EG_MATCH; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EG_MATCH); @@ -4026,8 +4004,6 @@ TARGET(CHECK_EXC_MATCH) { int opcode = CHECK_EXC_MATCH; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EXC_MATCH); @@ -4461,8 +4437,6 @@ TARGET(CONVERT_VALUE) { int opcode = CONVERT_VALUE; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CONVERT_VALUE); @@ -4493,8 +4467,6 @@ TARGET(COPY) { int opcode = COPY; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY); @@ -4512,8 +4484,6 @@ TARGET(COPY_FREE_VARS) { int opcode = COPY_FREE_VARS; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY_FREE_VARS); @@ -4534,8 +4504,6 @@ TARGET(DELETE_ATTR) { int opcode = DELETE_ATTR; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_ATTR); @@ -4557,8 +4525,6 @@ TARGET(DELETE_DEREF) { int opcode = DELETE_DEREF; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_DEREF); @@ -4581,8 +4547,6 @@ TARGET(DELETE_FAST) { int opcode = DELETE_FAST; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_FAST); @@ -4607,8 +4571,6 @@ TARGET(DELETE_GLOBAL) { int opcode = DELETE_GLOBAL; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_GLOBAL); @@ -4633,8 +4595,6 @@ TARGET(DELETE_NAME) { int opcode = DELETE_NAME; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_NAME); @@ -4666,8 +4626,6 @@ TARGET(DELETE_SUBSCR) { int opcode = DELETE_SUBSCR; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_SUBSCR); @@ -4693,8 +4651,6 @@ TARGET(DICT_MERGE) { int opcode = DICT_MERGE; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_MERGE); @@ -4726,8 +4682,6 @@ TARGET(DICT_UPDATE) { int opcode = DICT_UPDATE; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_UPDATE); @@ -4798,8 +4752,6 @@ TARGET(END_FOR) { int opcode = END_FOR; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; next_instr += 1; INSTRUCTION_STATS(END_FOR); _PyStackRef value; @@ -4820,8 +4772,6 @@ TARGET(END_SEND) { int opcode = END_SEND; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_SEND); @@ -4878,8 +4828,6 @@ TARGET(EXIT_INIT_CHECK) { int opcode = EXIT_INIT_CHECK; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXIT_INIT_CHECK); @@ -4902,8 +4850,6 @@ TARGET(EXTENDED_ARG) { int opcode = EXTENDED_ARG; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXTENDED_ARG); @@ -4917,8 +4863,6 @@ TARGET(FORMAT_SIMPLE) { int opcode = FORMAT_SIMPLE; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_SIMPLE); @@ -4956,8 +4900,6 @@ TARGET(FORMAT_WITH_SPEC) { int opcode = FORMAT_WITH_SPEC; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_WITH_SPEC); @@ -5280,8 +5222,6 @@ TARGET(GET_AITER) { int opcode = GET_AITER; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AITER); @@ -5333,8 +5273,6 @@ TARGET(GET_ANEXT) { int opcode = GET_ANEXT; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ANEXT); @@ -5357,8 +5295,6 @@ TARGET(GET_AWAITABLE) { int opcode = GET_AWAITABLE; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AWAITABLE); @@ -5380,8 +5316,6 @@ TARGET(GET_ITER) { int opcode = GET_ITER; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ITER); @@ -5404,8 +5338,6 @@ TARGET(GET_LEN) { int opcode = GET_LEN; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_LEN); @@ -5433,8 +5365,6 @@ TARGET(GET_YIELD_FROM_ITER) { int opcode = GET_YIELD_FROM_ITER; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_YIELD_FROM_ITER); @@ -5480,8 +5410,6 @@ TARGET(IMPORT_FROM) { int opcode = IMPORT_FROM; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_FROM); @@ -5505,8 +5433,6 @@ TARGET(IMPORT_NAME) { int opcode = IMPORT_NAME; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_NAME); @@ -6691,8 +6617,6 @@ TARGET(INTERPRETER_EXIT) { int opcode = INTERPRETER_EXIT; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INTERPRETER_EXIT); @@ -6714,8 +6638,6 @@ TARGET(IS_OP) { int opcode = IS_OP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IS_OP); @@ -6855,8 +6777,6 @@ TARGET(JUMP_BACKWARD_NO_INTERRUPT) { int opcode = JUMP_BACKWARD_NO_INTERRUPT; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); @@ -6873,8 +6793,6 @@ TARGET(JUMP_BACKWARD_NO_JIT) { int opcode = JUMP_BACKWARD_NO_JIT; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); @@ -6909,8 +6827,6 @@ TARGET(JUMP_FORWARD) { int opcode = JUMP_FORWARD; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_FORWARD); @@ -6921,8 +6837,6 @@ TARGET(LIST_APPEND) { int opcode = LIST_APPEND; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_APPEND); @@ -6943,8 +6857,6 @@ TARGET(LIST_EXTEND) { int opcode = LIST_EXTEND; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_EXTEND); @@ -7904,8 +7816,6 @@ TARGET(LOAD_BUILD_CLASS) { int opcode = LOAD_BUILD_CLASS; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_BUILD_CLASS); @@ -7934,8 +7844,6 @@ TARGET(LOAD_COMMON_CONSTANT) { int opcode = LOAD_COMMON_CONSTANT; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); @@ -7995,8 +7903,6 @@ TARGET(LOAD_CONST_IMMORTAL) { int opcode = LOAD_CONST_IMMORTAL; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST_IMMORTAL); @@ -8014,8 +7920,6 @@ TARGET(LOAD_CONST_MORTAL) { int opcode = LOAD_CONST_MORTAL; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST_MORTAL); @@ -8032,8 +7936,6 @@ TARGET(LOAD_DEREF) { int opcode = LOAD_DEREF; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_DEREF); @@ -8056,8 +7958,6 @@ TARGET(LOAD_FAST) { int opcode = LOAD_FAST; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST); @@ -8073,8 +7973,6 @@ TARGET(LOAD_FAST_AND_CLEAR) { int opcode = LOAD_FAST_AND_CLEAR; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); @@ -8090,8 +7988,6 @@ TARGET(LOAD_FAST_CHECK) { int opcode = LOAD_FAST_CHECK; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_CHECK); @@ -8116,8 +8012,6 @@ TARGET(LOAD_FAST_LOAD_FAST) { int opcode = LOAD_FAST_LOAD_FAST; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); @@ -8137,8 +8031,6 @@ TARGET(LOAD_FROM_DICT_OR_DEREF) { int opcode = LOAD_FROM_DICT_OR_DEREF; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); @@ -8182,8 +8074,6 @@ TARGET(LOAD_FROM_DICT_OR_GLOBALS) { int opcode = LOAD_FROM_DICT_OR_GLOBALS; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); @@ -8462,8 +8352,6 @@ TARGET(LOAD_LOCALS) { int opcode = LOAD_LOCALS; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_LOCALS); @@ -8486,8 +8374,6 @@ TARGET(LOAD_NAME) { int opcode = LOAD_NAME; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_NAME); @@ -8509,8 +8395,6 @@ TARGET(LOAD_SMALL_INT) { int opcode = LOAD_SMALL_INT; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SMALL_INT); @@ -8527,8 +8411,6 @@ TARGET(LOAD_SPECIAL) { int opcode = LOAD_SPECIAL; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SPECIAL); @@ -8792,8 +8674,6 @@ TARGET(MAKE_CELL) { int opcode = MAKE_CELL; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_CELL); @@ -8815,8 +8695,6 @@ TARGET(MAKE_FUNCTION) { int opcode = MAKE_FUNCTION; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_FUNCTION); @@ -8848,8 +8726,6 @@ TARGET(MAP_ADD) { int opcode = MAP_ADD; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAP_ADD); @@ -8881,8 +8757,6 @@ TARGET(MATCH_CLASS) { int opcode = MATCH_CLASS; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_CLASS); @@ -8925,8 +8799,6 @@ TARGET(MATCH_KEYS) { int opcode = MATCH_KEYS; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_KEYS); @@ -8953,8 +8825,6 @@ TARGET(MATCH_MAPPING) { int opcode = MATCH_MAPPING; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_MAPPING); @@ -8972,8 +8842,6 @@ TARGET(MATCH_SEQUENCE) { int opcode = MATCH_SEQUENCE; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_SEQUENCE); @@ -8991,8 +8859,6 @@ TARGET(NOP) { int opcode = NOP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOP); @@ -9002,8 +8868,6 @@ TARGET(NOT_TAKEN) { int opcode = NOT_TAKEN; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOT_TAKEN); @@ -9013,8 +8877,6 @@ TARGET(POP_EXCEPT) { int opcode = POP_EXCEPT; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_EXCEPT); @@ -9034,8 +8896,6 @@ TARGET(POP_ITER) { int opcode = POP_ITER; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_ITER); @@ -9162,8 +9022,6 @@ TARGET(POP_TOP) { int opcode = POP_TOP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_TOP); @@ -9178,8 +9036,6 @@ TARGET(PUSH_EXC_INFO) { int opcode = PUSH_EXC_INFO; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_EXC_INFO); @@ -9207,8 +9063,6 @@ TARGET(PUSH_NULL) { int opcode = PUSH_NULL; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_NULL); @@ -9299,8 +9153,6 @@ TARGET(RESERVED) { int opcode = RESERVED; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESERVED); @@ -9424,8 +9276,6 @@ TARGET(RETURN_GENERATOR) { int opcode = RETURN_GENERATOR; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); @@ -9463,8 +9313,6 @@ TARGET(RETURN_VALUE) { int opcode = RETURN_VALUE; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_VALUE); @@ -9661,8 +9509,6 @@ TARGET(SETUP_ANNOTATIONS) { int opcode = SETUP_ANNOTATIONS; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SETUP_ANNOTATIONS); @@ -9708,8 +9554,6 @@ TARGET(SET_ADD) { int opcode = SET_ADD; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_ADD); @@ -9733,8 +9577,6 @@ TARGET(SET_FUNCTION_ATTRIBUTE) { int opcode = SET_FUNCTION_ATTRIBUTE; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); @@ -9761,8 +9603,6 @@ TARGET(SET_UPDATE) { int opcode = SET_UPDATE; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_UPDATE); @@ -10056,8 +9896,6 @@ TARGET(STORE_DEREF) { int opcode = STORE_DEREF; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_DEREF); @@ -10075,8 +9913,6 @@ TARGET(STORE_FAST) { int opcode = STORE_FAST; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST); @@ -10095,8 +9931,6 @@ TARGET(STORE_FAST_LOAD_FAST) { int opcode = STORE_FAST_LOAD_FAST; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); @@ -10118,8 +9952,6 @@ TARGET(STORE_FAST_STORE_FAST) { int opcode = STORE_FAST_STORE_FAST; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_STORE_FAST); @@ -10149,8 +9981,6 @@ TARGET(STORE_GLOBAL) { int opcode = STORE_GLOBAL; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_GLOBAL); @@ -10172,8 +10002,6 @@ TARGET(STORE_NAME) { int opcode = STORE_NAME; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_NAME); @@ -10212,8 +10040,6 @@ TARGET(STORE_SLICE) { int opcode = STORE_SLICE; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_SLICE); @@ -10418,8 +10244,6 @@ TARGET(SWAP) { int opcode = SWAP; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SWAP); @@ -10658,8 +10482,6 @@ TARGET(UNARY_INVERT) { int opcode = UNARY_INVERT; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_INVERT); @@ -10681,8 +10503,6 @@ TARGET(UNARY_NEGATIVE) { int opcode = UNARY_NEGATIVE; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NEGATIVE); @@ -10704,8 +10524,6 @@ TARGET(UNARY_NOT) { int opcode = UNARY_NOT; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NOT); @@ -10722,8 +10540,6 @@ TARGET(UNPACK_EX) { int opcode = UNPACK_EX; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNPACK_EX); @@ -10912,8 +10728,6 @@ TARGET(WITH_EXCEPT_START) { int opcode = WITH_EXCEPT_START; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(WITH_EXCEPT_START); @@ -10970,8 +10784,6 @@ TARGET(YIELD_VALUE) { int opcode = YIELD_VALUE; (void)(opcode); - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(YIELD_VALUE); diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 503516f95f046c..78f7b35b90c2f0 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -137,7 +137,7 @@ def uses_this(inst: Instruction) -> bool: if (tkn.kind == "IDENTIFIER" and (tkn.text in {"DEOPT_IF", "EXIT_IF"})): return True - return False, False + return False UNKNOWN_OPCODE_HANDLER ="""\ From 61d9247918e9ef88c98c3c2679670d39d9c39e1c Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 6 Feb 2025 22:53:52 +0800 Subject: [PATCH 110/110] Fix non-computed-gotos --- Lib/test/test_generated_cases.py | 260 ++++- Python/generated_cases.c.h | 1110 +++++++++++++++++----- Tools/cases_generator/tier1_generator.py | 7 +- 3 files changed, 1102 insertions(+), 275 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index e426311c1560b5..35ad9ebbe5a1a9 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -304,7 +304,10 @@ def test_inst_no_args(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -324,7 +327,10 @@ def test_inst_one_pop(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -347,7 +353,10 @@ def test_inst_one_push(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -371,7 +380,10 @@ def test_inst_one_push_one_pop(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -396,7 +408,10 @@ def test_binary_op(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -424,7 +439,10 @@ def test_overlap(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -454,7 +472,10 @@ def test_predictions(self): """ output = """ TARGET(OP1) { - int opcode = OP1; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP1; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -467,7 +488,10 @@ def test_predictions(self): } TARGET(OP3) { - int opcode = OP3; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP3; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -503,7 +527,10 @@ def test_sync_sp(self): """ output = """ TARGET(A) { - int opcode = A; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = A; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -522,7 +549,10 @@ def test_sync_sp(self): } TARGET(B) { - int opcode = B; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = B; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -561,7 +591,10 @@ def test_error_if_plain(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -582,7 +615,10 @@ def test_error_if_plain_with_comment(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -607,7 +643,10 @@ def test_error_if_pop(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -640,7 +679,10 @@ def test_error_if_pop_with_result(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -669,7 +711,10 @@ def test_cache_effect(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -698,7 +743,10 @@ def test_suppress_dispatch(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -731,7 +779,10 @@ def test_macro_instruction(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 6; @@ -770,7 +821,10 @@ def test_macro_instruction(self): } TARGET(OP1) { - int opcode = OP1; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP1; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -790,7 +844,10 @@ def test_macro_instruction(self): } TARGET(OP3) { - int opcode = OP3; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP3; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 6; @@ -823,7 +880,10 @@ def test_unused_caches(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 4; @@ -847,7 +907,10 @@ def test_pseudo_instruction_no_flags(self): """ output = """ TARGET(OP1) { - int opcode = OP1; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP1; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -868,7 +931,10 @@ def test_pseudo_instruction_with_flags(self): """ output = """ TARGET(OP1) { - int opcode = OP1; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP1; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -892,7 +958,10 @@ def test_pseudo_instruction_as_sequence(self): """ output = """ TARGET(OP1) { - int opcode = OP1; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP1; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -901,7 +970,10 @@ def test_pseudo_instruction_as_sequence(self): } TARGET(OP2) { - int opcode = OP2; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP2; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -920,7 +992,10 @@ def test_array_input(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -945,7 +1020,10 @@ def test_array_output(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -975,7 +1053,10 @@ def test_array_input_output(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1001,7 +1082,10 @@ def test_array_error_if(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1029,7 +1113,10 @@ def test_cond_effect(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1072,7 +1159,10 @@ def test_macro_cond_effect(self): """ output = """ TARGET(M) { - int opcode = M; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = M; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1118,7 +1208,10 @@ def test_macro_push_push(self): """ output = """ TARGET(M) { - int opcode = M; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = M; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1153,7 +1246,10 @@ def test_override_inst(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1176,7 +1272,10 @@ def test_override_op(self): """ output = """ TARGET(M) { - int opcode = M; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = M; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1195,7 +1294,10 @@ def test_annotated_inst(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1215,7 +1317,10 @@ def test_annotated_op(self): """ output = """ TARGET(M) { - int opcode = M; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = M; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1253,7 +1358,10 @@ def test_array_of_one(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1277,7 +1385,10 @@ def test_pointer_to_stackref(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1317,7 +1428,10 @@ def test_unused_named_values(self): """ output = """ TARGET(INST) { - int opcode = INST; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INST; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1345,7 +1459,10 @@ def test_used_unused_used(self): """ output = """ TARGET(TEST) { - int opcode = TEST; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TEST; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1387,7 +1504,10 @@ def test_unused_used_used(self): """ output = """ TARGET(TEST) { - int opcode = TEST; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TEST; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1428,7 +1548,10 @@ def test_flush(self): """ output = """ TARGET(TEST) { - int opcode = TEST; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TEST; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1478,7 +1601,10 @@ def test_pop_on_error_peeks(self): """ output = """ TARGET(TEST) { - int opcode = TEST; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TEST; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1528,7 +1654,10 @@ def test_push_then_error(self): output = """ TARGET(TEST) { - int opcode = TEST; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TEST; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1571,7 +1700,10 @@ def test_error_if_true(self): """ output = """ TARGET(OP1) { - int opcode = OP1; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP1; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1580,7 +1712,10 @@ def test_error_if_true(self): } TARGET(OP2) { - int opcode = OP2; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP2; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1639,7 +1774,10 @@ def test_stack_save_reload(self): output = """ TARGET(BALANCED) { - int opcode = BALANCED; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BALANCED; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1661,7 +1799,10 @@ def test_stack_reload_only(self): output = """ TARGET(BALANCED) { - int opcode = BALANCED; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BALANCED; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1684,7 +1825,10 @@ def test_stack_save_only(self): output = """ TARGET(BALANCED) { - int opcode = BALANCED; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BALANCED; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -1708,7 +1852,10 @@ def test_instruction_size_macro(self): output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1745,7 +1892,10 @@ def test_escaping_call_next_to_cmacro(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1781,7 +1931,10 @@ def test_pystackref_frompyobject_new_next_to_cmacro(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1814,7 +1967,10 @@ def test_pop_input(self): """ output = """ TARGET(OP) { - int opcode = OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 69a77e7cd71e34..7927fa7db95c20 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -19,7 +19,10 @@ TARGET(BINARY_OP) { - int opcode = BINARY_OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 6; @@ -73,7 +76,10 @@ } TARGET(BINARY_OP_ADD_FLOAT) { - int opcode = BINARY_OP_ADD_FLOAT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_ADD_FLOAT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -125,7 +131,10 @@ } TARGET(BINARY_OP_ADD_INT) { - int opcode = BINARY_OP_ADD_INT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_ADD_INT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -176,7 +185,10 @@ } TARGET(BINARY_OP_ADD_UNICODE) { - int opcode = BINARY_OP_ADD_UNICODE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_ADD_UNICODE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -227,7 +239,10 @@ } TARGET(BINARY_OP_EXTEND) { - int opcode = BINARY_OP_EXTEND; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_EXTEND; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -281,7 +296,10 @@ } TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { - int opcode = BINARY_OP_INPLACE_ADD_UNICODE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_INPLACE_ADD_UNICODE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -362,7 +380,10 @@ } TARGET(BINARY_OP_MULTIPLY_FLOAT) { - int opcode = BINARY_OP_MULTIPLY_FLOAT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_MULTIPLY_FLOAT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -414,7 +435,10 @@ } TARGET(BINARY_OP_MULTIPLY_INT) { - int opcode = BINARY_OP_MULTIPLY_INT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_MULTIPLY_INT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -465,7 +489,10 @@ } TARGET(BINARY_OP_SUBTRACT_FLOAT) { - int opcode = BINARY_OP_SUBTRACT_FLOAT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_SUBTRACT_FLOAT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -517,7 +544,10 @@ } TARGET(BINARY_OP_SUBTRACT_INT) { - int opcode = BINARY_OP_SUBTRACT_INT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_SUBTRACT_INT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -568,7 +598,10 @@ } TARGET(BINARY_SLICE) { - int opcode = BINARY_SLICE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_SLICE; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -626,7 +659,10 @@ } TARGET(BINARY_SUBSCR) { - int opcode = BINARY_SUBSCR; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_SUBSCR; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; @@ -677,7 +713,10 @@ } TARGET(BINARY_SUBSCR_DICT) { - int opcode = BINARY_SUBSCR_DICT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_SUBSCR_DICT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -722,7 +761,10 @@ } TARGET(BINARY_SUBSCR_GETITEM) { - int opcode = BINARY_SUBSCR_GETITEM; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_SUBSCR_GETITEM; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -805,7 +847,10 @@ } TARGET(BINARY_SUBSCR_LIST_INT) { - int opcode = BINARY_SUBSCR_LIST_INT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_SUBSCR_LIST_INT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -873,7 +918,10 @@ } TARGET(BINARY_SUBSCR_STR_INT) { - int opcode = BINARY_SUBSCR_STR_INT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_SUBSCR_STR_INT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -933,7 +981,10 @@ } TARGET(BINARY_SUBSCR_TUPLE_INT) { - int opcode = BINARY_SUBSCR_TUPLE_INT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_SUBSCR_TUPLE_INT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -989,7 +1040,10 @@ } TARGET(BUILD_LIST) { - int opcode = BUILD_LIST; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BUILD_LIST; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1009,7 +1063,10 @@ } TARGET(BUILD_MAP) { - int opcode = BUILD_MAP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BUILD_MAP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1049,7 +1106,10 @@ } TARGET(BUILD_SET) { - int opcode = BUILD_SET; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BUILD_SET; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1095,7 +1155,10 @@ } TARGET(BUILD_SLICE) { - int opcode = BUILD_SLICE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BUILD_SLICE; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1123,7 +1186,10 @@ } TARGET(BUILD_STRING) { - int opcode = BUILD_STRING; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BUILD_STRING; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1158,7 +1224,10 @@ } TARGET(BUILD_TUPLE) { - int opcode = BUILD_TUPLE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BUILD_TUPLE; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1178,7 +1247,10 @@ } TARGET(CACHE) { - int opcode = CACHE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CACHE; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -1189,7 +1261,10 @@ } TARGET(CALL) { - int opcode = CALL; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 4; @@ -1356,7 +1431,10 @@ } TARGET(CALL_ALLOC_AND_ENTER_INIT) { - int opcode = CALL_ALLOC_AND_ENTER_INIT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_ALLOC_AND_ENTER_INIT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -1480,7 +1558,10 @@ } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - int opcode = CALL_BOUND_METHOD_EXACT_ARGS; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_BOUND_METHOD_EXACT_ARGS; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -1617,7 +1698,10 @@ } TARGET(CALL_BOUND_METHOD_GENERAL) { - int opcode = CALL_BOUND_METHOD_GENERAL; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_BOUND_METHOD_GENERAL; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -1737,7 +1821,10 @@ } TARGET(CALL_BUILTIN_CLASS) { - int opcode = CALL_BUILTIN_CLASS; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_BUILTIN_CLASS; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -1827,7 +1914,10 @@ } TARGET(CALL_BUILTIN_FAST) { - int opcode = CALL_BUILTIN_FAST; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_BUILTIN_FAST; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -1923,7 +2013,10 @@ } TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { - int opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -2020,7 +2113,10 @@ } TARGET(CALL_BUILTIN_O) { - int opcode = CALL_BUILTIN_O; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_BUILTIN_O; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -2114,7 +2210,10 @@ } TARGET(CALL_FUNCTION_EX) { - int opcode = CALL_FUNCTION_EX; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_FUNCTION_EX; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -2303,7 +2402,10 @@ } TARGET(CALL_INTRINSIC_1) { - int opcode = CALL_INTRINSIC_1; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_INTRINSIC_1; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -2325,7 +2427,10 @@ } TARGET(CALL_INTRINSIC_2) { - int opcode = CALL_INTRINSIC_2; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_INTRINSIC_2; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -2354,7 +2459,10 @@ } TARGET(CALL_ISINSTANCE) { - int opcode = CALL_ISINSTANCE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_ISINSTANCE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -2413,7 +2521,10 @@ } TARGET(CALL_KW) { - int opcode = CALL_KW; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_KW; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 4; @@ -2579,7 +2690,10 @@ } TARGET(CALL_KW_BOUND_METHOD) { - int opcode = CALL_KW_BOUND_METHOD; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_KW_BOUND_METHOD; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -2710,7 +2824,10 @@ } TARGET(CALL_KW_NON_PY) { - int opcode = CALL_KW_NON_PY; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_KW_NON_PY; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -2820,7 +2937,10 @@ } TARGET(CALL_KW_PY) { - int opcode = CALL_KW_PY; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_KW_PY; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -2925,7 +3045,10 @@ } TARGET(CALL_LEN) { - int opcode = CALL_LEN; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_LEN; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -2990,7 +3113,10 @@ } TARGET(CALL_LIST_APPEND) { - int opcode = CALL_LIST_APPEND; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_LIST_APPEND; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -3052,7 +3178,10 @@ } TARGET(CALL_METHOD_DESCRIPTOR_FAST) { - int opcode = CALL_METHOD_DESCRIPTOR_FAST; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_METHOD_DESCRIPTOR_FAST; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -3154,7 +3283,10 @@ } TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - int opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -3256,7 +3388,10 @@ } TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { - int opcode = CALL_METHOD_DESCRIPTOR_NOARGS; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_METHOD_DESCRIPTOR_NOARGS; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -3358,7 +3493,10 @@ } TARGET(CALL_METHOD_DESCRIPTOR_O) { - int opcode = CALL_METHOD_DESCRIPTOR_O; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_METHOD_DESCRIPTOR_O; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -3462,7 +3600,10 @@ } TARGET(CALL_NON_PY_GENERAL) { - int opcode = CALL_NON_PY_GENERAL; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_NON_PY_GENERAL; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -3562,7 +3703,10 @@ } TARGET(CALL_PY_EXACT_ARGS) { - int opcode = CALL_PY_EXACT_ARGS; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_PY_EXACT_ARGS; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -3671,7 +3815,10 @@ } TARGET(CALL_PY_GENERAL) { - int opcode = CALL_PY_GENERAL; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_PY_GENERAL; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -3765,7 +3912,10 @@ } TARGET(CALL_STR_1) { - int opcode = CALL_STR_1; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_STR_1; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -3836,7 +3986,10 @@ } TARGET(CALL_TUPLE_1) { - int opcode = CALL_TUPLE_1; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_TUPLE_1; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -3907,7 +4060,10 @@ } TARGET(CALL_TYPE_1) { - int opcode = CALL_TYPE_1; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_TYPE_1; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -3949,7 +4105,10 @@ } TARGET(CHECK_EG_MATCH) { - int opcode = CHECK_EG_MATCH; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CHECK_EG_MATCH; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -4002,7 +4161,10 @@ } TARGET(CHECK_EXC_MATCH) { - int opcode = CHECK_EXC_MATCH; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CHECK_EXC_MATCH; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -4032,7 +4194,10 @@ } TARGET(CLEANUP_THROW) { - int opcode = CLEANUP_THROW; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CLEANUP_THROW; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -4078,7 +4243,10 @@ } TARGET(COMPARE_OP) { - int opcode = COMPARE_OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = COMPARE_OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; @@ -4145,7 +4313,10 @@ } TARGET(COMPARE_OP_FLOAT) { - int opcode = COMPARE_OP_FLOAT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = COMPARE_OP_FLOAT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -4195,7 +4366,10 @@ } TARGET(COMPARE_OP_INT) { - int opcode = COMPARE_OP_INT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = COMPARE_OP_INT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -4257,7 +4431,10 @@ } TARGET(COMPARE_OP_STR) { - int opcode = COMPARE_OP_STR; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = COMPARE_OP_STR; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -4308,7 +4485,10 @@ } TARGET(CONTAINS_OP) { - int opcode = CONTAINS_OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CONTAINS_OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; @@ -4358,7 +4538,10 @@ } TARGET(CONTAINS_OP_DICT) { - int opcode = CONTAINS_OP_DICT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CONTAINS_OP_DICT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -4396,7 +4579,10 @@ } TARGET(CONTAINS_OP_SET) { - int opcode = CONTAINS_OP_SET; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CONTAINS_OP_SET; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -4435,7 +4621,10 @@ } TARGET(CONVERT_VALUE) { - int opcode = CONVERT_VALUE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CONVERT_VALUE; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -4465,7 +4654,10 @@ } TARGET(COPY) { - int opcode = COPY; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = COPY; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -4482,7 +4674,10 @@ } TARGET(COPY_FREE_VARS) { - int opcode = COPY_FREE_VARS; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = COPY_FREE_VARS; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -4502,7 +4697,10 @@ } TARGET(DELETE_ATTR) { - int opcode = DELETE_ATTR; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DELETE_ATTR; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -4523,7 +4721,10 @@ } TARGET(DELETE_DEREF) { - int opcode = DELETE_DEREF; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DELETE_DEREF; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -4545,7 +4746,10 @@ } TARGET(DELETE_FAST) { - int opcode = DELETE_FAST; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DELETE_FAST; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -4569,7 +4773,10 @@ } TARGET(DELETE_GLOBAL) { - int opcode = DELETE_GLOBAL; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DELETE_GLOBAL; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -4593,7 +4800,10 @@ } TARGET(DELETE_NAME) { - int opcode = DELETE_NAME; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DELETE_NAME; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -4624,7 +4834,10 @@ } TARGET(DELETE_SUBSCR) { - int opcode = DELETE_SUBSCR; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DELETE_SUBSCR; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -4649,7 +4862,10 @@ } TARGET(DICT_MERGE) { - int opcode = DICT_MERGE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DICT_MERGE; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -4680,7 +4896,10 @@ } TARGET(DICT_UPDATE) { - int opcode = DICT_UPDATE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DICT_UPDATE; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -4715,7 +4934,10 @@ } TARGET(END_ASYNC_FOR) { - int opcode = END_ASYNC_FOR; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = END_ASYNC_FOR; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -4750,7 +4972,10 @@ } TARGET(END_FOR) { - int opcode = END_FOR; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = END_FOR; (void)(opcode); next_instr += 1; INSTRUCTION_STATS(END_FOR); @@ -4770,7 +4995,10 @@ } TARGET(END_SEND) { - int opcode = END_SEND; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = END_SEND; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -4790,7 +5018,10 @@ } TARGET(ENTER_EXECUTOR) { - int opcode = ENTER_EXECUTOR; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = ENTER_EXECUTOR; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -4826,7 +5057,10 @@ } TARGET(EXIT_INIT_CHECK) { - int opcode = EXIT_INIT_CHECK; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = EXIT_INIT_CHECK; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -4848,7 +5082,10 @@ } TARGET(EXTENDED_ARG) { - int opcode = EXTENDED_ARG; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = EXTENDED_ARG; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -4861,7 +5098,10 @@ } TARGET(FORMAT_SIMPLE) { - int opcode = FORMAT_SIMPLE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = FORMAT_SIMPLE; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -4898,7 +5138,10 @@ } TARGET(FORMAT_WITH_SPEC) { - int opcode = FORMAT_WITH_SPEC; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = FORMAT_WITH_SPEC; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -4924,7 +5167,10 @@ } TARGET(FOR_ITER) { - int opcode = FOR_ITER; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = FOR_ITER; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; @@ -4988,7 +5234,10 @@ } TARGET(FOR_ITER_GEN) { - int opcode = FOR_ITER_GEN; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = FOR_ITER_GEN; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -5052,7 +5301,10 @@ } TARGET(FOR_ITER_LIST) { - int opcode = FOR_ITER_LIST; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = FOR_ITER_LIST; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -5111,7 +5363,10 @@ } TARGET(FOR_ITER_RANGE) { - int opcode = FOR_ITER_RANGE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = FOR_ITER_RANGE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -5164,7 +5419,10 @@ } TARGET(FOR_ITER_TUPLE) { - int opcode = FOR_ITER_TUPLE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = FOR_ITER_TUPLE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -5220,7 +5478,10 @@ } TARGET(GET_AITER) { - int opcode = GET_AITER; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = GET_AITER; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -5271,7 +5532,10 @@ } TARGET(GET_ANEXT) { - int opcode = GET_ANEXT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = GET_ANEXT; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -5293,7 +5557,10 @@ } TARGET(GET_AWAITABLE) { - int opcode = GET_AWAITABLE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = GET_AWAITABLE; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -5314,7 +5581,10 @@ } TARGET(GET_ITER) { - int opcode = GET_ITER; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = GET_ITER; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -5336,7 +5606,10 @@ } TARGET(GET_LEN) { - int opcode = GET_LEN; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = GET_LEN; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -5363,7 +5636,10 @@ } TARGET(GET_YIELD_FROM_ITER) { - int opcode = GET_YIELD_FROM_ITER; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = GET_YIELD_FROM_ITER; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -5408,7 +5684,10 @@ } TARGET(IMPORT_FROM) { - int opcode = IMPORT_FROM; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = IMPORT_FROM; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -5431,7 +5710,10 @@ } TARGET(IMPORT_NAME) { - int opcode = IMPORT_NAME; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = IMPORT_NAME; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -5460,7 +5742,10 @@ } TARGET(INSTRUMENTED_CALL) { - int opcode = INSTRUMENTED_CALL; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_CALL; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -5638,7 +5923,10 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - int opcode = INSTRUMENTED_CALL_FUNCTION_EX; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_CALL_FUNCTION_EX; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -5827,7 +6115,10 @@ } TARGET(INSTRUMENTED_CALL_KW) { - int opcode = INSTRUMENTED_CALL_KW; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_CALL_KW; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -6002,7 +6293,10 @@ } TARGET(INSTRUMENTED_END_FOR) { - int opcode = INSTRUMENTED_END_FOR; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_END_FOR; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -6029,7 +6323,10 @@ } TARGET(INSTRUMENTED_END_SEND) { - int opcode = INSTRUMENTED_END_SEND; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_END_SEND; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -6061,7 +6358,10 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - int opcode = INSTRUMENTED_FOR_ITER; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_FOR_ITER; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -6101,7 +6401,10 @@ } TARGET(INSTRUMENTED_INSTRUCTION) { - int opcode = INSTRUMENTED_INSTRUCTION; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_INSTRUCTION; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -6125,7 +6428,10 @@ } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - int opcode = INSTRUMENTED_JUMP_BACKWARD; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_JUMP_BACKWARD; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -6154,7 +6460,10 @@ } TARGET(INSTRUMENTED_JUMP_FORWARD) { - int opcode = INSTRUMENTED_JUMP_FORWARD; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_JUMP_FORWARD; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -6166,7 +6475,10 @@ } TARGET(INSTRUMENTED_LINE) { - int opcode = INSTRUMENTED_LINE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_LINE; (void)(opcode); _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; @@ -6207,7 +6519,10 @@ } TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { - int opcode = INSTRUMENTED_LOAD_SUPER_ATTR; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_LOAD_SUPER_ATTR; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -6300,7 +6615,10 @@ } TARGET(INSTRUMENTED_NOT_TAKEN) { - int opcode = INSTRUMENTED_NOT_TAKEN; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_NOT_TAKEN; (void)(opcode); _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; @@ -6314,7 +6632,10 @@ } TARGET(INSTRUMENTED_POP_ITER) { - int opcode = INSTRUMENTED_POP_ITER; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_POP_ITER; (void)(opcode); _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; @@ -6334,7 +6655,10 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -6353,7 +6677,10 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - int opcode = INSTRUMENTED_POP_JUMP_IF_NONE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_POP_JUMP_IF_NONE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -6376,7 +6703,10 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -6397,7 +6727,10 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -6416,7 +6749,10 @@ } TARGET(INSTRUMENTED_RESUME) { - int opcode = INSTRUMENTED_RESUME; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_RESUME; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -6497,7 +6833,10 @@ } TARGET(INSTRUMENTED_RETURN_VALUE) { - int opcode = INSTRUMENTED_RETURN_VALUE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_RETURN_VALUE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -6545,7 +6884,10 @@ } TARGET(INSTRUMENTED_YIELD_VALUE) { - int opcode = INSTRUMENTED_YIELD_VALUE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_YIELD_VALUE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -6615,7 +6957,10 @@ } TARGET(INTERPRETER_EXIT) { - int opcode = INTERPRETER_EXIT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INTERPRETER_EXIT; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -6636,7 +6981,10 @@ } TARGET(IS_OP) { - int opcode = IS_OP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = IS_OP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -6657,7 +7005,10 @@ } TARGET(JUMP_BACKWARD) { - int opcode = JUMP_BACKWARD; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = JUMP_BACKWARD; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; @@ -6704,7 +7055,10 @@ } TARGET(JUMP_BACKWARD_JIT) { - int opcode = JUMP_BACKWARD_JIT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = JUMP_BACKWARD_JIT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -6775,7 +7129,10 @@ } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - int opcode = JUMP_BACKWARD_NO_INTERRUPT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = JUMP_BACKWARD_NO_INTERRUPT; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -6791,7 +7148,10 @@ } TARGET(JUMP_BACKWARD_NO_JIT) { - int opcode = JUMP_BACKWARD_NO_JIT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = JUMP_BACKWARD_NO_JIT; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; @@ -6825,7 +7185,10 @@ } TARGET(JUMP_FORWARD) { - int opcode = JUMP_FORWARD; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = JUMP_FORWARD; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -6835,7 +7198,10 @@ } TARGET(LIST_APPEND) { - int opcode = LIST_APPEND; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LIST_APPEND; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -6855,7 +7221,10 @@ } TARGET(LIST_EXTEND) { - int opcode = LIST_EXTEND; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LIST_EXTEND; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -6894,7 +7263,10 @@ } TARGET(LOAD_ATTR) { - int opcode = LOAD_ATTR; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 10; @@ -6976,7 +7348,10 @@ } TARGET(LOAD_ATTR_CLASS) { - int opcode = LOAD_ATTR_CLASS; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_CLASS; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -7026,7 +7401,10 @@ } TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) { - int opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -7086,7 +7464,10 @@ } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -7142,7 +7523,10 @@ } TARGET(LOAD_ATTR_INSTANCE_VALUE) { - int opcode = LOAD_ATTR_INSTANCE_VALUE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_INSTANCE_VALUE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -7217,7 +7601,10 @@ } TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - int opcode = LOAD_ATTR_METHOD_LAZY_DICT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_METHOD_LAZY_DICT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -7272,7 +7659,10 @@ } TARGET(LOAD_ATTR_METHOD_NO_DICT) { - int opcode = LOAD_ATTR_METHOD_NO_DICT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_METHOD_NO_DICT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -7316,7 +7706,10 @@ } TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - int opcode = LOAD_ATTR_METHOD_WITH_VALUES; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_METHOD_WITH_VALUES; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -7382,7 +7775,10 @@ } TARGET(LOAD_ATTR_MODULE) { - int opcode = LOAD_ATTR_MODULE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_MODULE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -7459,7 +7855,10 @@ } TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { - int opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -7498,7 +7897,10 @@ } TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - int opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -7558,7 +7960,10 @@ } TARGET(LOAD_ATTR_PROPERTY) { - int opcode = LOAD_ATTR_PROPERTY; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_PROPERTY; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -7651,7 +8056,10 @@ } TARGET(LOAD_ATTR_SLOT) { - int opcode = LOAD_ATTR_SLOT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_SLOT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -7712,7 +8120,10 @@ } TARGET(LOAD_ATTR_WITH_HINT) { - int opcode = LOAD_ATTR_WITH_HINT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_WITH_HINT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -7814,7 +8225,10 @@ } TARGET(LOAD_BUILD_CLASS) { - int opcode = LOAD_BUILD_CLASS; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_BUILD_CLASS; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -7842,7 +8256,10 @@ } TARGET(LOAD_COMMON_CONSTANT) { - int opcode = LOAD_COMMON_CONSTANT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_COMMON_CONSTANT; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -7866,7 +8283,10 @@ } TARGET(LOAD_CONST) { - int opcode = LOAD_CONST; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_CONST; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -7901,7 +8321,10 @@ } TARGET(LOAD_CONST_IMMORTAL) { - int opcode = LOAD_CONST_IMMORTAL; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_CONST_IMMORTAL; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -7918,7 +8341,10 @@ } TARGET(LOAD_CONST_MORTAL) { - int opcode = LOAD_CONST_MORTAL; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_CONST_MORTAL; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -7934,7 +8360,10 @@ } TARGET(LOAD_DEREF) { - int opcode = LOAD_DEREF; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_DEREF; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -7956,7 +8385,10 @@ } TARGET(LOAD_FAST) { - int opcode = LOAD_FAST; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_FAST; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -7971,7 +8403,10 @@ } TARGET(LOAD_FAST_AND_CLEAR) { - int opcode = LOAD_FAST_AND_CLEAR; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_FAST_AND_CLEAR; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -7986,7 +8421,10 @@ } TARGET(LOAD_FAST_CHECK) { - int opcode = LOAD_FAST_CHECK; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_FAST_CHECK; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -8010,7 +8448,10 @@ } TARGET(LOAD_FAST_LOAD_FAST) { - int opcode = LOAD_FAST_LOAD_FAST; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_FAST_LOAD_FAST; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -8029,7 +8470,10 @@ } TARGET(LOAD_FROM_DICT_OR_DEREF) { - int opcode = LOAD_FROM_DICT_OR_DEREF; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_FROM_DICT_OR_DEREF; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -8072,7 +8516,10 @@ } TARGET(LOAD_FROM_DICT_OR_GLOBALS) { - int opcode = LOAD_FROM_DICT_OR_GLOBALS; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_FROM_DICT_OR_GLOBALS; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -8150,7 +8597,10 @@ } TARGET(LOAD_GLOBAL) { - int opcode = LOAD_GLOBAL; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_GLOBAL; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 5; @@ -8202,7 +8652,10 @@ } TARGET(LOAD_GLOBAL_BUILTIN) { - int opcode = LOAD_GLOBAL_BUILTIN; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_GLOBAL_BUILTIN; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -8284,7 +8737,10 @@ } TARGET(LOAD_GLOBAL_MODULE) { - int opcode = LOAD_GLOBAL_MODULE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_GLOBAL_MODULE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -8350,7 +8806,10 @@ } TARGET(LOAD_LOCALS) { - int opcode = LOAD_LOCALS; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_LOCALS; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -8372,7 +8831,10 @@ } TARGET(LOAD_NAME) { - int opcode = LOAD_NAME; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_NAME; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -8393,7 +8855,10 @@ } TARGET(LOAD_SMALL_INT) { - int opcode = LOAD_SMALL_INT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_SMALL_INT; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -8409,7 +8874,10 @@ } TARGET(LOAD_SPECIAL) { - int opcode = LOAD_SPECIAL; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_SPECIAL; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -8448,7 +8916,10 @@ } TARGET(LOAD_SUPER_ATTR) { - int opcode = LOAD_SUPER_ATTR; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_SUPER_ATTR; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; @@ -8558,7 +9029,10 @@ } TARGET(LOAD_SUPER_ATTR_ATTR) { - int opcode = LOAD_SUPER_ATTR_ATTR; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_SUPER_ATTR_ATTR; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -8607,7 +9081,10 @@ } TARGET(LOAD_SUPER_ATTR_METHOD) { - int opcode = LOAD_SUPER_ATTR_METHOD; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_SUPER_ATTR_METHOD; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -8672,7 +9149,10 @@ } TARGET(MAKE_CELL) { - int opcode = MAKE_CELL; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = MAKE_CELL; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -8693,7 +9173,10 @@ } TARGET(MAKE_FUNCTION) { - int opcode = MAKE_FUNCTION; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = MAKE_FUNCTION; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -8724,7 +9207,10 @@ } TARGET(MAP_ADD) { - int opcode = MAP_ADD; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = MAP_ADD; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -8755,7 +9241,10 @@ } TARGET(MATCH_CLASS) { - int opcode = MATCH_CLASS; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = MATCH_CLASS; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -8797,7 +9286,10 @@ } TARGET(MATCH_KEYS) { - int opcode = MATCH_KEYS; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = MATCH_KEYS; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -8823,7 +9315,10 @@ } TARGET(MATCH_MAPPING) { - int opcode = MATCH_MAPPING; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = MATCH_MAPPING; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -8840,7 +9335,10 @@ } TARGET(MATCH_SEQUENCE) { - int opcode = MATCH_SEQUENCE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = MATCH_SEQUENCE; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -8857,7 +9355,10 @@ } TARGET(NOP) { - int opcode = NOP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = NOP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -8866,7 +9367,10 @@ } TARGET(NOT_TAKEN) { - int opcode = NOT_TAKEN; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = NOT_TAKEN; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -8875,7 +9379,10 @@ } TARGET(POP_EXCEPT) { - int opcode = POP_EXCEPT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = POP_EXCEPT; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -8894,7 +9401,10 @@ } TARGET(POP_ITER) { - int opcode = POP_ITER; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = POP_ITER; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -8908,7 +9418,10 @@ } TARGET(POP_JUMP_IF_FALSE) { - int opcode = POP_JUMP_IF_FALSE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = POP_JUMP_IF_FALSE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -8928,7 +9441,10 @@ } TARGET(POP_JUMP_IF_NONE) { - int opcode = POP_JUMP_IF_NONE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = POP_JUMP_IF_NONE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -8964,7 +9480,10 @@ } TARGET(POP_JUMP_IF_NOT_NONE) { - int opcode = POP_JUMP_IF_NOT_NONE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = POP_JUMP_IF_NOT_NONE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -9000,7 +9519,10 @@ } TARGET(POP_JUMP_IF_TRUE) { - int opcode = POP_JUMP_IF_TRUE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = POP_JUMP_IF_TRUE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -9020,7 +9542,10 @@ } TARGET(POP_TOP) { - int opcode = POP_TOP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = POP_TOP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -9034,7 +9559,10 @@ } TARGET(PUSH_EXC_INFO) { - int opcode = PUSH_EXC_INFO; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = PUSH_EXC_INFO; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -9061,7 +9589,10 @@ } TARGET(PUSH_NULL) { - int opcode = PUSH_NULL; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = PUSH_NULL; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -9075,7 +9606,10 @@ } TARGET(RAISE_VARARGS) { - int opcode = RAISE_VARARGS; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = RAISE_VARARGS; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -9104,7 +9638,10 @@ } TARGET(RERAISE) { - int opcode = RERAISE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = RERAISE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -9151,7 +9688,10 @@ } TARGET(RESERVED) { - int opcode = RESERVED; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = RESERVED; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -9162,7 +9702,10 @@ } TARGET(RESUME) { - int opcode = RESUME; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = RESUME; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -9238,7 +9781,10 @@ } TARGET(RESUME_CHECK) { - int opcode = RESUME_CHECK; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = RESUME_CHECK; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -9274,7 +9820,10 @@ } TARGET(RETURN_GENERATOR) { - int opcode = RETURN_GENERATOR; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = RETURN_GENERATOR; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -9311,7 +9860,10 @@ } TARGET(RETURN_VALUE) { - int opcode = RETURN_VALUE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = RETURN_VALUE; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -9341,7 +9893,10 @@ } TARGET(SEND) { - int opcode = SEND; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = SEND; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; @@ -9439,7 +9994,10 @@ } TARGET(SEND_GEN) { - int opcode = SEND_GEN; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = SEND_GEN; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -9507,7 +10065,10 @@ } TARGET(SETUP_ANNOTATIONS) { - int opcode = SETUP_ANNOTATIONS; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = SETUP_ANNOTATIONS; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -9552,7 +10113,10 @@ } TARGET(SET_ADD) { - int opcode = SET_ADD; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = SET_ADD; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -9575,7 +10139,10 @@ } TARGET(SET_FUNCTION_ATTRIBUTE) { - int opcode = SET_FUNCTION_ATTRIBUTE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = SET_FUNCTION_ATTRIBUTE; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -9601,7 +10168,10 @@ } TARGET(SET_UPDATE) { - int opcode = SET_UPDATE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = SET_UPDATE; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -9624,7 +10194,10 @@ } TARGET(STORE_ATTR) { - int opcode = STORE_ATTR; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_ATTR; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 5; @@ -9673,7 +10246,10 @@ } TARGET(STORE_ATTR_INSTANCE_VALUE) { - int opcode = STORE_ATTR_INSTANCE_VALUE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_ATTR_INSTANCE_VALUE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -9747,7 +10323,10 @@ } TARGET(STORE_ATTR_SLOT) { - int opcode = STORE_ATTR_SLOT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_ATTR_SLOT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -9796,7 +10375,10 @@ } TARGET(STORE_ATTR_WITH_HINT) { - int opcode = STORE_ATTR_WITH_HINT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_ATTR_WITH_HINT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -9894,7 +10476,10 @@ } TARGET(STORE_DEREF) { - int opcode = STORE_DEREF; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_DEREF; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -9911,7 +10496,10 @@ } TARGET(STORE_FAST) { - int opcode = STORE_FAST; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_FAST; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -9929,7 +10517,10 @@ } TARGET(STORE_FAST_LOAD_FAST) { - int opcode = STORE_FAST_LOAD_FAST; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_FAST_LOAD_FAST; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -9950,7 +10541,10 @@ } TARGET(STORE_FAST_STORE_FAST) { - int opcode = STORE_FAST_STORE_FAST; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_FAST_STORE_FAST; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -9979,7 +10573,10 @@ } TARGET(STORE_GLOBAL) { - int opcode = STORE_GLOBAL; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_GLOBAL; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -10000,7 +10597,10 @@ } TARGET(STORE_NAME) { - int opcode = STORE_NAME; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_NAME; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -10038,7 +10638,10 @@ } TARGET(STORE_SLICE) { - int opcode = STORE_SLICE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_SLICE; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -10090,7 +10693,10 @@ } TARGET(STORE_SUBSCR) { - int opcode = STORE_SUBSCR; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_SUBSCR; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; @@ -10139,7 +10745,10 @@ } TARGET(STORE_SUBSCR_DICT) { - int opcode = STORE_SUBSCR_DICT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_SUBSCR_DICT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -10178,7 +10787,10 @@ } TARGET(STORE_SUBSCR_LIST_INT) { - int opcode = STORE_SUBSCR_LIST_INT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_SUBSCR_LIST_INT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -10242,7 +10854,10 @@ } TARGET(SWAP) { - int opcode = SWAP; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = SWAP; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -10259,7 +10874,10 @@ } TARGET(TO_BOOL) { - int opcode = TO_BOOL; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TO_BOOL; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 4; @@ -10303,7 +10921,10 @@ } TARGET(TO_BOOL_ALWAYS_TRUE) { - int opcode = TO_BOOL_ALWAYS_TRUE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TO_BOOL_ALWAYS_TRUE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -10338,7 +10959,10 @@ } TARGET(TO_BOOL_BOOL) { - int opcode = TO_BOOL_BOOL; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TO_BOOL_BOOL; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -10360,7 +10984,10 @@ } TARGET(TO_BOOL_INT) { - int opcode = TO_BOOL_INT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TO_BOOL_INT; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -10393,7 +11020,10 @@ } TARGET(TO_BOOL_LIST) { - int opcode = TO_BOOL_LIST; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TO_BOOL_LIST; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -10420,7 +11050,10 @@ } TARGET(TO_BOOL_NONE) { - int opcode = TO_BOOL_NONE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TO_BOOL_NONE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -10446,7 +11079,10 @@ } TARGET(TO_BOOL_STR) { - int opcode = TO_BOOL_STR; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TO_BOOL_STR; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -10480,7 +11116,10 @@ } TARGET(UNARY_INVERT) { - int opcode = UNARY_INVERT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNARY_INVERT; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -10501,7 +11140,10 @@ } TARGET(UNARY_NEGATIVE) { - int opcode = UNARY_NEGATIVE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNARY_NEGATIVE; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -10522,7 +11164,10 @@ } TARGET(UNARY_NOT) { - int opcode = UNARY_NOT; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNARY_NOT; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -10538,7 +11183,10 @@ } TARGET(UNPACK_EX) { - int opcode = UNPACK_EX; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNPACK_EX; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -10561,7 +11209,10 @@ } TARGET(UNPACK_SEQUENCE) { - int opcode = UNPACK_SEQUENCE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNPACK_SEQUENCE; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; @@ -10608,7 +11259,10 @@ } TARGET(UNPACK_SEQUENCE_LIST) { - int opcode = UNPACK_SEQUENCE_LIST; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNPACK_SEQUENCE_LIST; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -10653,7 +11307,10 @@ } TARGET(UNPACK_SEQUENCE_TUPLE) { - int opcode = UNPACK_SEQUENCE_TUPLE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNPACK_SEQUENCE_TUPLE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -10689,7 +11346,10 @@ } TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { - int opcode = UNPACK_SEQUENCE_TWO_TUPLE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNPACK_SEQUENCE_TWO_TUPLE; (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -10726,7 +11386,10 @@ } TARGET(WITH_EXCEPT_START) { - int opcode = WITH_EXCEPT_START; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = WITH_EXCEPT_START; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; @@ -10782,7 +11445,10 @@ } TARGET(YIELD_VALUE) { - int opcode = YIELD_VALUE; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = YIELD_VALUE; (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 78f7b35b90c2f0..979646a7ca6525 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -227,7 +227,12 @@ def generate_tier1_cases( for name, inst in sorted(analysis.instructions.items()): out.emit("\n") out.emit(f"TARGET({name}) {{\n") - out.emit(f"int opcode = {name};\n") + # We need to ifdef it because this breaks platforms + # without computed gotos/tail calling. + out.emit(f"#if defined(Py_TAIL_CALL_INTERP)\n") + out.emit(f"int opcode;\n") + out.emit(f"#endif\n") + out.emit(f"opcode = {name};\n") out.emit(f"(void)(opcode);\n") needs_this = uses_this(inst) unused_guard = "(void)this_instr;\n"