diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index a6bade2c044f80..cd6e9875d282d2 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -7,6 +7,7 @@ on: - 'Python/optimizer*.c' - 'Python/executor_cases.c.h' - 'Python/optimizer_cases.c.h' + - '**_testinternalcapi**' - '!Python/perf_jit_trampoline.c' - '!**/*.md' - '!**/*.ini' @@ -17,6 +18,7 @@ on: - 'Python/optimizer*.c' - 'Python/executor_cases.c.h' - 'Python/optimizer_cases.c.h' + - '**_testinternalcapi**' - '!Python/perf_jit_trampoline.c' - '!**/*.md' - '!**/*.ini' diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst index 313c534545a861..e23288c6a58590 100644 --- a/Doc/c-api/descriptor.rst +++ b/Doc/c-api/descriptor.rst @@ -10,11 +10,6 @@ found in the dictionary of type objects. .. XXX document these! -.. c:var:: PyTypeObject PyProperty_Type - - The type object for the built-in descriptor types. - - .. c:function:: PyObject* PyDescr_NewGetSet(PyTypeObject *type, struct PyGetSetDef *getset) @@ -74,9 +69,26 @@ found in the dictionary of type objects. .. c:function:: PyObject* PyWrapper_New(PyObject *, PyObject *) +.. c:macro:: PyDescr_COMMON + + This is a :term:`soft deprecated` macro including the common fields for a + descriptor object. + + This was included in Python's C API by mistake; do not use it in extensions. + For creating custom descriptor objects, create a class implementing the + descriptor protocol (:c:member:`~PyTypeObject.tp_descr_get` and + :c:member:`~PyTypeObject.tp_descr_set`). + + Built-in descriptors ^^^^^^^^^^^^^^^^^^^^ +.. c:var:: PyTypeObject PyProperty_Type + + The type object for property objects. This is the same object as + :class:`property` in the Python layer. + + .. c:var:: PyTypeObject PySuper_Type The type object for super objects. This is the same object as diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index d7fe9e2c9ec9b4..59af470f59ff34 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -793,6 +793,17 @@ Exception Classes Return :c:member:`~PyTypeObject.tp_name` of the exception class *ob*. +.. c:macro:: PyException_HEAD + + This is a :term:`soft deprecated` macro including the base fields for an + exception object. + + This was included in Python's C API by mistake and is not designed for use + in extensions. For creating custom exception objects, use + :c:func:`PyErr_NewException` or otherwise create a class inheriting from + :c:data:`PyExc_BaseException`. + + Exception Objects ================= diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index b4d19eb69511d2..0592221f15226e 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -115,8 +115,10 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) #define REF_IS_BORROWED 1 +#define REF_IS_INVALID 2 +#define REF_TAG_BITS 3 -#define JIT_BITS_TO_PTR_MASKED(REF) ((JitOptSymbol *)(((REF).bits) & (~REF_IS_BORROWED))) +#define JIT_BITS_TO_PTR_MASKED(REF) ((JitOptSymbol *)(((REF).bits) & (~REF_TAG_BITS))) static inline JitOptSymbol * PyJitRef_Unwrap(JitOptRef ref) @@ -133,6 +135,18 @@ PyJitRef_Wrap(JitOptSymbol *sym) return (JitOptRef){.bits=(uintptr_t)sym}; } +static inline JitOptRef +PyJitRef_WrapInvalid(void *ptr) +{ + return (JitOptRef){.bits=(uintptr_t)ptr | REF_IS_INVALID}; +} + +static inline bool +PyJitRef_IsInvalid(JitOptRef ref) +{ + return (ref.bits & REF_IS_INVALID) == REF_IS_INVALID; +} + static inline JitOptRef PyJitRef_StripReferenceInfo(JitOptRef ref) { @@ -225,7 +239,7 @@ PyAPI_FUNC(int) _PyDumpExecutors(FILE *out); PyAPI_FUNC(void) _Py_ClearExecutorDeletionList(PyInterpreterState *interp); #endif -int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int stop_tracing_opcode); +PyAPI_FUNC(int) _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int stop_tracing_opcode); PyAPI_FUNC(int) _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, @@ -233,7 +247,7 @@ _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, int oparg, _PyExecutorObject *current_executor); -void _PyJit_FinalizeTracing(PyThreadState *tstate, int err); +PyAPI_FUNC(void) _PyJit_FinalizeTracing(PyThreadState *tstate, int err); void _PyJit_TracerFree(_PyThreadStateImpl *_tstate); void _PyJit_Tracer_InvalidateDependency(PyThreadState *old_tstate, void *obj); diff --git a/Include/internal/pycore_optimizer_types.h b/Include/internal/pycore_optimizer_types.h index 0a193268c4d618..6501ce869c1425 100644 --- a/Include/internal/pycore_optimizer_types.h +++ b/Include/internal/pycore_optimizer_types.h @@ -8,6 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include #include "pycore_uop.h" // UOP_MAX_TRACE_LENGTH // Holds locals, stack, locals, stack ... (in that order) @@ -128,6 +129,8 @@ typedef struct _JitOptContext { JitOptRef *n_consumed; JitOptRef *limit; JitOptRef locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; + _PyUOpInstruction *out_buffer; + int out_len; } JitOptContext; diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index dc4444717b4e87..24a40416c2191b 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -60,6 +60,7 @@ typedef struct _PyJitTracerState { _PyJitTracerTranslatorState translator_state; JitOptContext opt_context; _PyUOpInstruction code_buffer[UOP_MAX_TRACE_LENGTH]; + _PyUOpInstruction out_buffer[UOP_MAX_TRACE_LENGTH]; } _PyJitTracerState; #endif diff --git a/Python/optimizer.c b/Python/optimizer.c index 9892a9731e24f0..ab0ef3db4e4882 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1535,6 +1535,7 @@ uop_optimize( if (length <= 0) { return length; } + buffer = _tstate->jit_tracer_state->out_buffer; } assert(length < UOP_MAX_TRACE_LENGTH/2); assert(length >= 1); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index e855df4977acf8..d635ebabf9007a 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -39,6 +39,7 @@ #ifdef Py_DEBUG extern const char *_PyUOpName(int index); extern void _PyUOpPrint(const _PyUOpInstruction *uop); + extern void _PyUOpSymPrint(JitOptRef ref); static const char *const DEBUG_ENV = "PYTHON_OPT_DEBUG"; static inline int get_lltrace(void) { char *uop_debug = Py_GETENV(DEBUG_ENV); @@ -50,6 +51,38 @@ } #define DPRINTF(level, ...) \ if (get_lltrace() >= (level)) { printf(__VA_ARGS__); } + + + +static void +dump_abstract_stack(_Py_UOpsAbstractFrame *frame, JitOptRef *stack_pointer) +{ + JitOptRef *stack_base = frame->stack; + JitOptRef *locals_base = frame->locals; + printf(" locals=["); + for (JitOptRef *ptr = locals_base; ptr < stack_base; ptr++) { + if (ptr != locals_base) { + printf(", "); + } + _PyUOpSymPrint(*ptr); + } + printf("]\n"); + if (stack_pointer < stack_base) { + printf(" stack=%d\n", (int)(stack_pointer - stack_base)); + } + else { + printf(" stack=["); + for (JitOptRef *ptr = stack_base; ptr < stack_pointer; ptr++) { + if (ptr != stack_base) { + printf(", "); + } + _PyUOpSymPrint(*ptr); + } + printf("]\n"); + } + fflush(stdout); +} + #else #define DPRINTF(level, ...) #endif @@ -143,6 +176,18 @@ incorrect_keys(PyObject *obj, uint32_t version) #define STACK_LEVEL() ((int)(stack_pointer - ctx->frame->stack)) #define STACK_SIZE() ((int)(ctx->frame->stack_len)) +static inline int +is_terminator_uop(const _PyUOpInstruction *uop) +{ + int opcode = uop->opcode; + return ( + opcode == _EXIT_TRACE || + opcode == _JUMP_TO_TOP || + opcode == _DYNAMIC_EXIT || + opcode == _DEOPT + ); +} + #define CURRENT_FRAME_IS_INIT_SHIM() (ctx->frame->code == ((PyCodeObject *)&_Py_InitCleanup)) #define GETLOCAL(idx) ((ctx->frame->locals[idx])) @@ -152,6 +197,22 @@ incorrect_keys(PyObject *obj, uint32_t version) (INST)->oparg = ARG; \ (INST)->operand0 = OPERAND; +#define ADD_OP(OP, ARG, OPERAND) add_op(ctx, this_instr, (OP), (ARG), (OPERAND)) + +static inline void +add_op(JitOptContext *ctx, _PyUOpInstruction *this_instr, + uint16_t opcode, uint16_t oparg, uintptr_t operand0) +{ + _PyUOpInstruction *out = &ctx->out_buffer[ctx->out_len]; + out->opcode = (opcode); + out->format = this_instr->format; + out->oparg = (oparg); + out->target = this_instr->target; + out->operand0 = (operand0); + out->operand1 = this_instr->operand1; + ctx->out_len++; +} + /* Shortened forms for convenience, used in optimizer_bytecodes.c */ #define sym_is_not_null _Py_uop_sym_is_not_null #define sym_is_const _Py_uop_sym_is_const @@ -219,7 +280,7 @@ optimize_to_bool( bool insert_mode) { if (sym_matches_type(value, &PyBool_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); *result_ptr = value; return 1; } @@ -229,7 +290,7 @@ optimize_to_bool( int opcode = insert_mode ? _INSERT_1_LOAD_CONST_INLINE_BORROW : _POP_TOP_LOAD_CONST_INLINE_BORROW; - REPLACE_OP(this_instr, opcode, 0, (uintptr_t)load); + ADD_OP(opcode, 0, (uintptr_t)load); *result_ptr = sym_new_const(ctx, load); return 1; } @@ -237,9 +298,9 @@ optimize_to_bool( } static void -eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit) +eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, bool exit) { - REPLACE_OP(this_instr, _POP_TOP, 0, 0); + ADD_OP(_POP_TOP, 0, 0); if (exit) { REPLACE_OP((this_instr+1), _EXIT_TRACE, 0, 0); this_instr[1].target = this_instr->target; @@ -256,7 +317,7 @@ lookup_attr(JitOptContext *ctx, _PyBloomFilter *dependencies, _PyUOpInstruction PyObject *lookup = _PyType_Lookup(type, name); if (lookup) { int opcode = _Py_IsImmortal(lookup) ? immortal : mortal; - REPLACE_OP(this_instr, opcode, 0, (uintptr_t)lookup); + ADD_OP(opcode, 0, (uintptr_t)lookup); PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); return sym_new_const(ctx, lookup); @@ -349,6 +410,8 @@ optimize_uops( JitOptContext *ctx = &tstate->jit_tracer_state->opt_context; uint32_t opcode = UINT16_MAX; + ctx->out_buffer = tstate->jit_tracer_state->out_buffer; + // Make sure that watchers are set up PyInterpreterState *interp = _PyInterpreterState_GET(); if (interp->dict_state.watchers[GLOBALS_WATCHER_ID] == NULL) { @@ -365,6 +428,8 @@ optimize_uops( ctx->curr_frame_depth++; ctx->frame = frame; + ctx->out_len = 0; + _PyUOpInstruction *this_instr = NULL; JitOptRef *stack_pointer = ctx->frame->stack_pointer; @@ -383,7 +448,10 @@ optimize_uops( if (get_lltrace() >= 3) { printf("%4d abs: ", (int)(this_instr - trace)); _PyUOpPrint(this_instr); - printf(" "); + printf(" \n"); + if (get_lltrace() >= 5 && !CURRENT_FRAME_IS_INIT_SHIM()) { + dump_abstract_stack(ctx->frame, stack_pointer); + } } #endif @@ -395,6 +463,10 @@ optimize_uops( DPRINTF(1, "\nUnknown opcode in abstract interpreter\n"); Py_UNREACHABLE(); } + // If no ADD_OP was called during this iteration, copy the original instruction + if (ctx->out_len == i) { + ctx->out_buffer[ctx->out_len++] = *this_instr; + } assert(ctx->frame != NULL); if (!CURRENT_FRAME_IS_INIT_SHIM()) { DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); @@ -423,7 +495,21 @@ optimize_uops( /* Either reached the end or cannot optimize further, but there * would be no benefit in retrying later */ _Py_uop_abstractcontext_fini(ctx); - return trace_len; + // Check that the trace ends with a proper terminator + if (ctx->out_len > 0) { + _PyUOpInstruction *last_uop = &ctx->out_buffer[ctx->out_len - 1]; + if (!is_terminator_uop(last_uop)) { + // Copy remaining uops from original trace until we find a terminator + for (int i = ctx->out_len; i < trace_len; i++) { + ctx->out_buffer[ctx->out_len++] = trace[i]; + if (is_terminator_uop(&trace[i])) { + break; + } + } + } + } + + return ctx->out_len; error: DPRINTF(3, "\n"); @@ -595,7 +681,7 @@ _Py_uop_analyze_and_optimize( assert(length > 0); - length = remove_unneeded_uops(buffer, length); + length = remove_unneeded_uops(tstate->jit_tracer_state->out_buffer, length); assert(length > 0); OPT_STAT_INC(optimizer_successes); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 1a64810b50a3a4..876ba7c6de7482 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -48,7 +48,7 @@ optimize_to_bool( bool insert_mode); extern void -eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit); +eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, bool exit); extern PyCodeObject *get_code(_PyUOpInstruction *op); @@ -139,11 +139,11 @@ dummy_func(void) { op(_GUARD_TOS_INT, (value -- value)) { if (sym_is_compact_int(value)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { if (sym_get_type(value) == &PyLong_Type) { - REPLACE_OP(this_instr, _GUARD_TOS_OVERFLOWED, 0, 0); + ADD_OP(_GUARD_TOS_OVERFLOWED, 0, 0); } sym_set_compact_int(value); } @@ -151,11 +151,11 @@ dummy_func(void) { op(_GUARD_NOS_INT, (left, unused -- left, unused)) { if (sym_is_compact_int(left)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { if (sym_get_type(left) == &PyLong_Type) { - REPLACE_OP(this_instr, _GUARD_NOS_OVERFLOWED, 0, 0); + ADD_OP(_GUARD_NOS_OVERFLOWED, 0, 0); } sym_set_compact_int(left); } @@ -165,7 +165,7 @@ dummy_func(void) { PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version); if (type) { if (type == sym_get_const(ctx, owner)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { sym_set_const(owner, type); @@ -176,7 +176,7 @@ dummy_func(void) { op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { assert(type_version); if (sym_matches_type_version(owner, type_version)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { // add watcher so that whenever the type changes we invalidate this PyTypeObject *type = _PyType_LookupByVersion(type_version); @@ -198,14 +198,14 @@ dummy_func(void) { op(_GUARD_TOS_FLOAT, (value -- value)) { if (sym_matches_type(value, &PyFloat_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(value, &PyFloat_Type); } op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) { if (sym_matches_type(left, &PyFloat_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(left, &PyFloat_Type); } @@ -340,7 +340,7 @@ dummy_func(void) { } f->locals[0] = container; f->locals[1] = sub; - new_frame = PyJitRef_Wrap((JitOptSymbol *)f); + new_frame = PyJitRef_WrapInvalid(f); } op(_BINARY_OP_SUBSCR_STR_INT, (str_st, sub_st -- res, s, i)) { @@ -363,7 +363,7 @@ dummy_func(void) { assert(index >= 0); int tuple_length = sym_tuple_length(tuple_st); if (tuple_length != -1 && index < tuple_length) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } } } @@ -438,7 +438,7 @@ dummy_func(void) { op(_GUARD_NOS_UNICODE, (nos, unused -- nos, unused)) { if (sym_matches_type(nos, &PyUnicode_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyUnicode_Type); } @@ -449,7 +449,7 @@ dummy_func(void) { op(_GUARD_TOS_UNICODE, (value -- value)) { if (sym_matches_type(value, &PyUnicode_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(value, &PyUnicode_Type); } @@ -560,7 +560,7 @@ dummy_func(void) { op(_LOAD_CONST, (-- value)) { PyCodeObject *co = get_current_code_object(ctx); PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); - REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); value = PyJitRef_Borrow(sym_new_const(ctx, val)); } @@ -568,7 +568,7 @@ dummy_func(void) { PyObject *val = PyLong_FromLong(oparg); assert(val); assert(_Py_IsImmortal(val)); - REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); value = PyJitRef_Borrow(sym_new_const(ctx, val)); } @@ -606,34 +606,34 @@ dummy_func(void) { if (PyJitRef_IsBorrowed(value) || sym_is_immortal(PyJitRef_Unwrap(value)) || sym_is_null(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } else if (typ == &PyLong_Type) { - REPLACE_OP(this_instr, _POP_TOP_INT, 0, 0); + ADD_OP(_POP_TOP_INT, 0, 0); } else if (typ == &PyFloat_Type) { - REPLACE_OP(this_instr, _POP_TOP_FLOAT, 0, 0); + ADD_OP(_POP_TOP_FLOAT, 0, 0); } else if (typ == &PyUnicode_Type) { - REPLACE_OP(this_instr, _POP_TOP_UNICODE, 0, 0); + ADD_OP(_POP_TOP_UNICODE, 0, 0); } } op(_POP_TOP_INT, (value --)) { if (PyJitRef_IsBorrowed(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } } op(_POP_TOP_FLOAT, (value --)) { if (PyJitRef_IsBorrowed(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } } op(_POP_TOP_UNICODE, (value --)) { if (PyJitRef_IsBorrowed(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } } @@ -686,11 +686,11 @@ dummy_func(void) { op (_PUSH_NULL_CONDITIONAL, ( -- null[oparg & 1])) { if (oparg & 1) { - REPLACE_OP(this_instr, _PUSH_NULL, 0, 0); + ADD_OP(_PUSH_NULL, 0, 0); null[0] = sym_new_null(ctx); } else { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } } @@ -784,7 +784,7 @@ dummy_func(void) { break; } f->locals[0] = owner; - new_frame = PyJitRef_Wrap((JitOptSymbol *)f); + new_frame = PyJitRef_WrapInvalid(f); } op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { @@ -795,8 +795,8 @@ dummy_func(void) { op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { assert(PyFunction_Check(sym_get_const(ctx, callable))); - REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - this_instr->operand1 = (uintptr_t)sym_get_const(ctx, callable); + ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + ctx->out_buffer[ctx->out_len - 1].operand1 = (uintptr_t)sym_get_const(ctx, callable); } sym_set_type(callable, &PyFunction_Type); } @@ -805,8 +805,8 @@ dummy_func(void) { if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); assert(PyMethod_Check(method)); - REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - this_instr->operand1 = (uintptr_t)method->im_func; + ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + ctx->out_buffer[ctx->out_len - 1].operand1 = (uintptr_t)method->im_func; } sym_set_type(callable, &PyMethod_Type); } @@ -818,7 +818,7 @@ dummy_func(void) { PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); PyCodeObject *co = (PyCodeObject *)func->func_code; if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { - REPLACE_OP(this_instr, _NOP, 0 ,0); + ADD_OP(_NOP, 0 ,0); } } } @@ -848,9 +848,9 @@ dummy_func(void) { } if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args, argcount)); + new_frame = PyJitRef_WrapInvalid(frame_new(ctx, co, 0, args, argcount)); } else { - new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); + new_frame = PyJitRef_WrapInvalid(frame_new(ctx, co, 0, NULL, 0)); } } @@ -868,7 +868,7 @@ dummy_func(void) { break; } - new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); + new_frame = PyJitRef_WrapInvalid(frame_new(ctx, co, 0, NULL, 0)); } op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { @@ -879,7 +879,7 @@ dummy_func(void) { break; } - new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); + new_frame = PyJitRef_WrapInvalid(frame_new(ctx, co, 0, NULL, 0)); } op(_PY_FRAME_EX, (func_st, null, callargs_st, kwargs_st -- ex_frame)) { @@ -890,7 +890,7 @@ dummy_func(void) { break; } - ex_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); + ex_frame = PyJitRef_WrapInvalid(frame_new(ctx, co, 0, NULL, 0)); } op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { @@ -914,7 +914,7 @@ dummy_func(void) { ctx->curr_frame_depth++; assert((this_instr + 1)->opcode == _PUSH_FRAME); PyCodeObject *co = get_code_with_logging((this_instr + 1)); - init_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args-1, oparg+1)); + init_frame = PyJitRef_WrapInvalid(frame_new(ctx, co, 0, args-1, oparg+1)); } op(_RETURN_VALUE, (retval -- res)) { @@ -1007,7 +1007,7 @@ dummy_func(void) { break; } new_frame->stack[0] = sym_new_const(ctx, Py_None); - gen_frame = PyJitRef_Wrap((JitOptSymbol *)new_frame); + gen_frame = PyJitRef_WrapInvalid(new_frame); } op(_SEND_GEN_FRAME, (unused, v -- unused, gen_frame)) { @@ -1023,7 +1023,7 @@ dummy_func(void) { break; } new_frame->stack[0] = PyJitRef_StripReferenceInfo(v); - gen_frame = PyJitRef_Wrap((JitOptSymbol *)new_frame); + gen_frame = PyJitRef_WrapInvalid(new_frame); } op(_CHECK_STACK_SPACE, (unused, unused, unused[oparg] -- unused, unused, unused[oparg])) { @@ -1081,7 +1081,7 @@ dummy_func(void) { op(_ITER_CHECK_TUPLE, (iter, null_or_index -- iter, null_or_index)) { if (sym_matches_type(iter, &PyTuple_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(iter, &PyTuple_Type); } @@ -1094,7 +1094,7 @@ dummy_func(void) { PyObject* type = (PyObject *)sym_get_type(arg); if (type) { res = sym_new_const(ctx, type); - REPLACE_OP(this_instr, _SHUFFLE_2_LOAD_CONST_INLINE_BORROW, 0, + ADD_OP(_SHUFFLE_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)type); } else { @@ -1132,7 +1132,7 @@ dummy_func(void) { out = Py_True; } sym_set_const(res, out); - REPLACE_OP(this_instr, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); + ADD_OP(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); } } @@ -1140,7 +1140,7 @@ dummy_func(void) { if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - eliminate_pop_guard(this_instr, value != Py_True); + eliminate_pop_guard(this_instr, ctx, value != Py_True); } sym_set_const(flag, Py_True); } @@ -1185,7 +1185,7 @@ dummy_func(void) { if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - eliminate_pop_guard(this_instr, value != Py_False); + eliminate_pop_guard(this_instr, ctx, value != Py_False); } sym_set_const(flag, Py_False); } @@ -1194,11 +1194,11 @@ dummy_func(void) { if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); assert(value != NULL); - eliminate_pop_guard(this_instr, !Py_IsNone(value)); + eliminate_pop_guard(this_instr, ctx, !Py_IsNone(value)); } else if (sym_has_type(val)) { assert(!sym_matches_type(val, &_PyNone_Type)); - eliminate_pop_guard(this_instr, true); + eliminate_pop_guard(this_instr, ctx, true); } sym_set_const(val, Py_None); } @@ -1207,11 +1207,11 @@ dummy_func(void) { if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); assert(value != NULL); - eliminate_pop_guard(this_instr, Py_IsNone(value)); + eliminate_pop_guard(this_instr, ctx, Py_IsNone(value)); } else if (sym_has_type(val)) { assert(!sym_matches_type(val, &_PyNone_Type)); - eliminate_pop_guard(this_instr, false); + eliminate_pop_guard(this_instr, ctx, false); } } @@ -1219,7 +1219,7 @@ dummy_func(void) { /* Setting the eval frame function invalidates * all executors, so no need to check dynamically */ if (_PyInterpreterState_GET()->eval_frame == NULL) { - REPLACE_OP(this_instr, _NOP, 0 ,0); + ADD_OP(_NOP, 0 ,0); } } @@ -1247,7 +1247,7 @@ dummy_func(void) { } op(_REPLACE_WITH_TRUE, (value -- res, v)) { - REPLACE_OP(this_instr, _INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); + ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); res = sym_new_const(ctx, Py_True); v = value; } @@ -1302,42 +1302,42 @@ dummy_func(void) { op(_GUARD_TOS_LIST, (tos -- tos)) { if (sym_matches_type(tos, &PyList_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(tos, &PyList_Type); } op(_GUARD_NOS_LIST, (nos, unused -- nos, unused)) { if (sym_matches_type(nos, &PyList_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyList_Type); } op(_GUARD_TOS_TUPLE, (tos -- tos)) { if (sym_matches_type(tos, &PyTuple_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(tos, &PyTuple_Type); } op(_GUARD_NOS_TUPLE, (nos, unused -- nos, unused)) { if (sym_matches_type(nos, &PyTuple_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyTuple_Type); } op(_GUARD_TOS_DICT, (tos -- tos)) { if (sym_matches_type(tos, &PyDict_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(tos, &PyDict_Type); } op(_GUARD_NOS_DICT, (nos, unused -- nos, unused)) { if (sym_matches_type(nos, &PyDict_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyDict_Type); } @@ -1346,48 +1346,48 @@ dummy_func(void) { if (sym_matches_type(tos, &PySet_Type) || sym_matches_type(tos, &PyFrozenSet_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } } op(_GUARD_NOS_NULL, (null, unused -- null, unused)) { if (sym_is_null(null)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_null(null); } op(_GUARD_NOS_NOT_NULL, (nos, unused -- nos, unused)) { if (sym_is_not_null(nos)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_non_null(nos); } op(_GUARD_THIRD_NULL, (null, unused, unused -- null, unused, unused)) { if (sym_is_null(null)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_null(null); } op(_GUARD_CALLABLE_TYPE_1, (callable, unused, unused -- callable, unused, unused)) { if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, (PyObject *)&PyType_Type); } op(_GUARD_CALLABLE_TUPLE_1, (callable, unused, unused -- callable, unused, unused)) { if (sym_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, (PyObject *)&PyTuple_Type); } op(_GUARD_CALLABLE_STR_1, (callable, unused, unused -- callable, unused, unused)) { if (sym_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, (PyObject *)&PyUnicode_Type); } @@ -1401,7 +1401,7 @@ dummy_func(void) { goto error; } if (_Py_IsImmortal(temp)) { - REPLACE_OP(this_instr, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, + ADD_OP(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); } res = sym_new_const(ctx, temp); @@ -1423,7 +1423,7 @@ dummy_func(void) { goto error; } if (_Py_IsImmortal(temp)) { - REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); } len = sym_new_const(ctx, temp); Py_DECREF(temp); @@ -1433,7 +1433,7 @@ dummy_func(void) { op(_GUARD_CALLABLE_LEN, (callable, unused, unused -- callable, unused, unused)) { PyObject *len = _PyInterpreterState_GET()->callable_cache.len; if (sym_get_const(ctx, callable) == len) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, len); } @@ -1441,7 +1441,7 @@ dummy_func(void) { op(_GUARD_CALLABLE_ISINSTANCE, (callable, unused, unused, unused -- callable, unused, unused, unused)) { PyObject *isinstance = _PyInterpreterState_GET()->callable_cache.isinstance; if (sym_get_const(ctx, callable) == isinstance) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, isinstance); } @@ -1449,7 +1449,7 @@ dummy_func(void) { op(_GUARD_CALLABLE_LIST_APPEND, (callable, unused, unused -- callable, unused, unused)) { PyObject *list_append = _PyInterpreterState_GET()->callable_cache.list_append; if (sym_get_const(ctx, callable) == list_append) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, list_append); } @@ -1485,7 +1485,7 @@ dummy_func(void) { ctx->frame->globals_watched = true; } if (ctx->frame->globals_checked_version == version) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } } } @@ -1546,7 +1546,7 @@ dummy_func(void) { ctx->frame->globals_watched = true; } if (ctx->frame->globals_checked_version != version && this_instr[-1].opcode == _NOP) { - REPLACE_OP(this_instr-1, _GUARD_GLOBALS_VERSION, 0, version); + REPLACE_OP(&ctx->out_buffer[ctx->out_len - 1], _GUARD_GLOBALS_VERSION, 0, version); ctx->frame->globals_checked_version = version; } if (ctx->frame->globals_checked_version == version) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index f3bc7213fcce3f..012fe16bfd9096 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -76,7 +76,7 @@ JitOptRef value; PyCodeObject *co = get_current_code_object(ctx); PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); - REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); value = PyJitRef_Borrow(sym_new_const(ctx, val)); CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; @@ -90,7 +90,7 @@ PyObject *val = PyLong_FromLong(oparg); assert(val); assert(_Py_IsImmortal(val)); - REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); value = PyJitRef_Borrow(sym_new_const(ctx, val)); CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; @@ -117,16 +117,16 @@ if (PyJitRef_IsBorrowed(value) || sym_is_immortal(PyJitRef_Unwrap(value)) || sym_is_null(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } else if (typ == &PyLong_Type) { - REPLACE_OP(this_instr, _POP_TOP_INT, 0, 0); + ADD_OP(_POP_TOP_INT, 0, 0); } else if (typ == &PyFloat_Type) { - REPLACE_OP(this_instr, _POP_TOP_FLOAT, 0, 0); + ADD_OP(_POP_TOP_FLOAT, 0, 0); } else if (typ == &PyUnicode_Type) { - REPLACE_OP(this_instr, _POP_TOP_UNICODE, 0, 0); + ADD_OP(_POP_TOP_UNICODE, 0, 0); } CHECK_STACK_BOUNDS(-1); stack_pointer += -1; @@ -145,7 +145,7 @@ JitOptRef value; value = stack_pointer[-1]; if (PyJitRef_IsBorrowed(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } CHECK_STACK_BOUNDS(-1); stack_pointer += -1; @@ -157,7 +157,7 @@ JitOptRef value; value = stack_pointer[-1]; if (PyJitRef_IsBorrowed(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } CHECK_STACK_BOUNDS(-1); stack_pointer += -1; @@ -169,7 +169,7 @@ JitOptRef value; value = stack_pointer[-1]; if (PyJitRef_IsBorrowed(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } CHECK_STACK_BOUNDS(-1); stack_pointer += -1; @@ -245,7 +245,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_1_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); @@ -295,7 +295,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } stack_pointer[-1] = res; @@ -353,7 +353,7 @@ JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyList_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyList_Type); break; @@ -363,7 +363,7 @@ JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PyList_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(tos, &PyList_Type); break; @@ -415,7 +415,7 @@ JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyUnicode_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyUnicode_Type); break; @@ -425,7 +425,7 @@ JitOptRef value; value = stack_pointer[-1]; if (sym_matches_type(value, &PyUnicode_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(value, &PyUnicode_Type); break; @@ -454,7 +454,7 @@ JitOptRef res; JitOptRef v; value = stack_pointer[-1]; - REPLACE_OP(this_instr, _INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); + ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); res = sym_new_const(ctx, Py_True); v = value; CHECK_STACK_BOUNDS(1); @@ -493,7 +493,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_1_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); @@ -522,11 +522,11 @@ JitOptRef left; left = stack_pointer[-2]; if (sym_is_compact_int(left)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { if (sym_get_type(left) == &PyLong_Type) { - REPLACE_OP(this_instr, _GUARD_NOS_OVERFLOWED, 0, 0); + ADD_OP(_GUARD_NOS_OVERFLOWED, 0, 0); } sym_set_compact_int(left); } @@ -537,11 +537,11 @@ JitOptRef value; value = stack_pointer[-1]; if (sym_is_compact_int(value)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { if (sym_get_type(value) == &PyLong_Type) { - REPLACE_OP(this_instr, _GUARD_TOS_OVERFLOWED, 0, 0); + ADD_OP(_GUARD_TOS_OVERFLOWED, 0, 0); } sym_set_compact_int(value); } @@ -600,7 +600,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); @@ -664,7 +664,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); @@ -728,7 +728,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); @@ -752,7 +752,7 @@ JitOptRef left; left = stack_pointer[-2]; if (sym_matches_type(left, &PyFloat_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(left, &PyFloat_Type); break; @@ -762,7 +762,7 @@ JitOptRef value; value = stack_pointer[-1]; if (sym_matches_type(value, &PyFloat_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(value, &PyFloat_Type); break; @@ -993,7 +993,7 @@ JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyTuple_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyTuple_Type); break; @@ -1003,7 +1003,7 @@ JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PyTuple_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(tos, &PyTuple_Type); break; @@ -1021,7 +1021,7 @@ assert(index >= 0); int tuple_length = sym_tuple_length(tuple_st); if (tuple_length != -1 && index < tuple_length) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } } break; @@ -1067,7 +1067,7 @@ JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyDict_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyDict_Type); break; @@ -1077,7 +1077,7 @@ JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PyDict_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(tos, &PyDict_Type); break; @@ -1131,7 +1131,7 @@ } f->locals[0] = container; f->locals[1] = sub; - new_frame = PyJitRef_Wrap((JitOptSymbol *)f); + new_frame = PyJitRef_WrapInvalid(f); CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = new_frame; stack_pointer += -2; @@ -1295,7 +1295,7 @@ break; } new_frame->stack[0] = PyJitRef_StripReferenceInfo(v); - gen_frame = PyJitRef_Wrap((JitOptSymbol *)new_frame); + gen_frame = PyJitRef_WrapInvalid(new_frame); stack_pointer[-1] = gen_frame; break; } @@ -1499,11 +1499,11 @@ JitOptRef *null; null = &stack_pointer[0]; if (oparg & 1) { - REPLACE_OP(this_instr, _PUSH_NULL, 0, 0); + ADD_OP(_PUSH_NULL, 0, 0); null[0] = sym_new_null(ctx); } else { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } CHECK_STACK_BOUNDS((oparg & 1)); stack_pointer += (oparg & 1); @@ -1528,7 +1528,7 @@ ctx->frame->globals_watched = true; } if (ctx->frame->globals_checked_version == version) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } } } @@ -1557,7 +1557,7 @@ ctx->frame->globals_watched = true; } if (ctx->frame->globals_checked_version != version && this_instr[-1].opcode == _NOP) { - REPLACE_OP(this_instr-1, _GUARD_GLOBALS_VERSION, 0, version); + REPLACE_OP(&ctx->out_buffer[ctx->out_len - 1], _GUARD_GLOBALS_VERSION, 0, version); ctx->frame->globals_checked_version = version; } if (ctx->frame->globals_checked_version == version) { @@ -1823,7 +1823,7 @@ uint32_t type_version = (uint32_t)this_instr->operand0; assert(type_version); if (sym_matches_type_version(owner, type_version)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { PyTypeObject *type = _PyType_LookupByVersion(type_version); if (type) { @@ -1937,7 +1937,7 @@ PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version); if (type) { if (type == sym_get_const(ctx, owner)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { sym_set_const(owner, type); @@ -1977,7 +1977,7 @@ break; } f->locals[0] = owner; - new_frame = PyJitRef_Wrap((JitOptSymbol *)f); + new_frame = PyJitRef_WrapInvalid(f); stack_pointer[-1] = new_frame; break; } @@ -2077,7 +2077,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(-1); @@ -2139,7 +2139,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); @@ -2203,7 +2203,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); @@ -2265,7 +2265,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); @@ -2345,7 +2345,7 @@ PyObject *result = sym_get_const(ctx, b); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); @@ -2371,7 +2371,7 @@ if (sym_matches_type(tos, &PySet_Type) || sym_matches_type(tos, &PyFrozenSet_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } break; } @@ -2481,7 +2481,7 @@ goto error; } if (_Py_IsImmortal(temp)) { - REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); } len = sym_new_const(ctx, temp); CHECK_STACK_BOUNDS(1); @@ -2606,7 +2606,7 @@ JitOptRef iter; iter = stack_pointer[-2]; if (sym_matches_type(iter, &PyTuple_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(iter, &PyTuple_Type); break; @@ -2662,7 +2662,7 @@ break; } new_frame->stack[0] = sym_new_const(ctx, Py_None); - gen_frame = PyJitRef_Wrap((JitOptSymbol *)new_frame); + gen_frame = PyJitRef_WrapInvalid(new_frame); CHECK_STACK_BOUNDS(1); stack_pointer[0] = gen_frame; stack_pointer += 1; @@ -2846,7 +2846,7 @@ ctx->done = true; break; } - new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); + new_frame = PyJitRef_WrapInvalid(frame_new(ctx, co, 0, NULL, 0)); CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; @@ -2860,8 +2860,8 @@ uint32_t func_version = (uint32_t)this_instr->operand0; if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { assert(PyFunction_Check(sym_get_const(ctx, callable))); - REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - this_instr->operand1 = (uintptr_t)sym_get_const(ctx, callable); + ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + ctx->out_buffer[ctx->out_len - 1].operand1 = (uintptr_t)sym_get_const(ctx, callable); } sym_set_type(callable, &PyFunction_Type); break; @@ -2878,8 +2878,8 @@ if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); assert(PyMethod_Check(method)); - REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - this_instr->operand1 = (uintptr_t)method->im_func; + ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + ctx->out_buffer[ctx->out_len - 1].operand1 = (uintptr_t)method->im_func; } sym_set_type(callable, &PyMethod_Type); break; @@ -2927,7 +2927,7 @@ case _CHECK_PEP_523: { if (_PyInterpreterState_GET()->eval_frame == NULL) { - REPLACE_OP(this_instr, _NOP, 0 ,0); + ADD_OP(_NOP, 0 ,0); } break; } @@ -2943,7 +2943,7 @@ PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); PyCodeObject *co = (PyCodeObject *)func->func_code; if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { - REPLACE_OP(this_instr, _NOP, 0 ,0); + ADD_OP(_NOP, 0 ,0); } } } @@ -2978,9 +2978,9 @@ argcount++; } if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args, argcount)); + new_frame = PyJitRef_WrapInvalid(frame_new(ctx, co, 0, args, argcount)); } else { - new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); + new_frame = PyJitRef_WrapInvalid(frame_new(ctx, co, 0, NULL, 0)); } CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = new_frame; @@ -3021,7 +3021,7 @@ JitOptRef null; null = stack_pointer[-2]; if (sym_is_null(null)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_null(null); break; @@ -3031,7 +3031,7 @@ JitOptRef nos; nos = stack_pointer[-2]; if (sym_is_not_null(nos)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_non_null(nos); break; @@ -3041,7 +3041,7 @@ JitOptRef null; null = stack_pointer[-3]; if (sym_is_null(null)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_null(null); break; @@ -3051,7 +3051,7 @@ JitOptRef callable; callable = stack_pointer[-3]; if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, (PyObject *)&PyType_Type); break; @@ -3065,8 +3065,8 @@ PyObject* type = (PyObject *)sym_get_type(arg); if (type) { res = sym_new_const(ctx, type); - REPLACE_OP(this_instr, _SHUFFLE_2_LOAD_CONST_INLINE_BORROW, 0, - (uintptr_t)type); + ADD_OP(_SHUFFLE_2_LOAD_CONST_INLINE_BORROW, 0, + (uintptr_t)type); } else { res = sym_new_not_null(ctx); @@ -3084,7 +3084,7 @@ JitOptRef callable; callable = stack_pointer[-3]; if (sym_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, (PyObject *)&PyUnicode_Type); break; @@ -3114,7 +3114,7 @@ JitOptRef callable; callable = stack_pointer[-3]; if (sym_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, (PyObject *)&PyTuple_Type); break; @@ -3175,7 +3175,7 @@ ctx->curr_frame_depth++; assert((this_instr + 1)->opcode == _PUSH_FRAME); PyCodeObject *co = get_code_with_logging((this_instr + 1)); - init_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args-1, oparg+1)); + init_frame = PyJitRef_WrapInvalid(frame_new(ctx, co, 0, args-1, oparg+1)); CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = init_frame; stack_pointer += -1 - oparg; @@ -3256,7 +3256,7 @@ callable = stack_pointer[-3]; PyObject *len = _PyInterpreterState_GET()->callable_cache.len; if (sym_get_const(ctx, callable) == len) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, len); break; @@ -3278,8 +3278,8 @@ goto error; } if (_Py_IsImmortal(temp)) { - REPLACE_OP(this_instr, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, - 0, (uintptr_t)temp); + ADD_OP(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW, + 0, (uintptr_t)temp); } res = sym_new_const(ctx, temp); CHECK_STACK_BOUNDS(-2); @@ -3302,7 +3302,7 @@ callable = stack_pointer[-4]; PyObject *isinstance = _PyInterpreterState_GET()->callable_cache.isinstance; if (sym_get_const(ctx, callable) == isinstance) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, isinstance); break; @@ -3323,7 +3323,7 @@ out = Py_True; } sym_set_const(res, out); - REPLACE_OP(this_instr, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); + ADD_OP(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); } CHECK_STACK_BOUNDS(-3); stack_pointer[-4] = res; @@ -3337,7 +3337,7 @@ callable = stack_pointer[-3]; PyObject *list_append = _PyInterpreterState_GET()->callable_cache.list_append; if (sym_get_const(ctx, callable) == list_append) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, list_append); break; @@ -3441,7 +3441,7 @@ ctx->done = true; break; } - new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); + new_frame = PyJitRef_WrapInvalid(frame_new(ctx, co, 0, NULL, 0)); CHECK_STACK_BOUNDS(-2 - oparg); stack_pointer[-3 - oparg] = new_frame; stack_pointer += -2 - oparg; @@ -3493,7 +3493,7 @@ ctx->done = true; break; } - ex_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); + ex_frame = PyJitRef_WrapInvalid(frame_new(ctx, co, 0, NULL, 0)); CHECK_STACK_BOUNDS(-3); stack_pointer[-4] = ex_frame; stack_pointer += -3; @@ -3631,7 +3631,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(-1); @@ -3718,7 +3718,7 @@ if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - eliminate_pop_guard(this_instr, value != Py_True); + eliminate_pop_guard(this_instr, ctx, value != Py_True); } sym_set_const(flag, Py_True); CHECK_STACK_BOUNDS(-1); @@ -3733,7 +3733,7 @@ if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - eliminate_pop_guard(this_instr, value != Py_False); + eliminate_pop_guard(this_instr, ctx, value != Py_False); } sym_set_const(flag, Py_False); CHECK_STACK_BOUNDS(-1); @@ -3748,11 +3748,11 @@ if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); assert(value != NULL); - eliminate_pop_guard(this_instr, !Py_IsNone(value)); + eliminate_pop_guard(this_instr, ctx, !Py_IsNone(value)); } else if (sym_has_type(val)) { assert(!sym_matches_type(val, &_PyNone_Type)); - eliminate_pop_guard(this_instr, true); + eliminate_pop_guard(this_instr, ctx, true); } sym_set_const(val, Py_None); CHECK_STACK_BOUNDS(-1); @@ -3767,11 +3767,11 @@ if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); assert(value != NULL); - eliminate_pop_guard(this_instr, Py_IsNone(value)); + eliminate_pop_guard(this_instr, ctx, Py_IsNone(value)); } else if (sym_has_type(val)) { assert(!sym_matches_type(val, &_PyNone_Type)); - eliminate_pop_guard(this_instr, false); + eliminate_pop_guard(this_instr, ctx, false); } CHECK_STACK_BOUNDS(-1); stack_pointer += -1; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index c6b54b9b58b795..5f5086d33b5c4c 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -68,6 +68,57 @@ static inline int get_lltrace(void) { } #define DPRINTF(level, ...) \ if (get_lltrace() >= (level)) { printf(__VA_ARGS__); } + +void +_PyUOpSymPrint(JitOptRef ref) +{ + if (PyJitRef_IsNull(ref)) { + printf(""); + return; + } + if (PyJitRef_IsInvalid(ref)) { + printf("", (void *)PyJitRef_Unwrap(ref)); + return; + } + JitOptSymbol *sym = PyJitRef_Unwrap(ref); + switch (sym->tag) { + case JIT_SYM_UNKNOWN_TAG: + printf("", (void *)sym); + break; + case JIT_SYM_NULL_TAG: + printf("", (void *)sym); + break; + case JIT_SYM_NON_NULL_TAG: + printf("", (void *)sym); + break; + case JIT_SYM_BOTTOM_TAG: + printf("", (void *)sym); + break; + case JIT_SYM_TYPE_VERSION_TAG: + printf("", sym->version.version, (void *)sym); + break; + case JIT_SYM_KNOWN_CLASS_TAG: + printf("<%s at %p>", sym->cls.type->tp_name, (void *)sym); + break; + case JIT_SYM_KNOWN_VALUE_TAG: + printf("<%s val=%p at %p>", Py_TYPE(sym->value.value)->tp_name, + (void *)sym->value.value, (void *)sym); + break; + case JIT_SYM_TUPLE_TAG: + printf("", sym->tuple.length, (void *)sym); + break; + case JIT_SYM_TRUTHINESS_TAG: + printf("", sym->truthiness.invert ? "!" : "", (void *)sym); + break; + case JIT_SYM_COMPACT_INT: + printf("", (void *)sym); + break; + default: + printf("", sym->tag, (void *)sym); + break; + } +} + #else #define DPRINTF(level, ...) #endif diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 392aad86549e62..65896221ba7a05 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -270,7 +270,7 @@ def replace_opcode_if_evaluates_pure( emitter.emit(f"PyObject *result = sym_get_const(ctx, {output_identifier.text});\n") emitter.emit(f"if (_Py_IsImmortal(result)) {{\n") emitter.emit(f"// Replace with {replacement_uop} since we have {input_desc} and an immortal result\n") - emitter.emit(f"REPLACE_OP(this_instr, {replacement_uop}, 0, (uintptr_t)result);\n") + emitter.emit(f"ADD_OP({replacement_uop}, 0, (uintptr_t)result);\n") emitter.emit("}\n") emitter.emit("}\n") diff --git a/Tools/check-c-api-docs/ignored_c_api.txt b/Tools/check-c-api-docs/ignored_c_api.txt index de78f6425803cb..e0b2670808c79c 100644 --- a/Tools/check-c-api-docs/ignored_c_api.txt +++ b/Tools/check-c-api-docs/ignored_c_api.txt @@ -66,7 +66,6 @@ PyCF_MASK PyCF_MASK_OBSOLETE PyCF_SOURCE_IS_UTF8 # cpython/descrobject.h -PyDescr_COMMON PyDescr_NAME PyDescr_TYPE PyWrapperFlag_KEYWORDS @@ -76,8 +75,6 @@ Py_UniversalNewlineFgets PyUnstable_CopyPerfMapFile PyUnstable_PerfTrampoline_CompileCode PyUnstable_PerfTrampoline_SetPersistAfterFork -# cpython/pyerrors.h -PyException_HEAD # cpython/pyframe.h PyUnstable_EXECUTABLE_KINDS PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION @@ -121,4 +118,4 @@ PY_MONITORING_EVENT_BRANCH PY_DEF_EVENT PY_FOREACH_DICT_EVENT # cpython/pystats.h -PYSTATS_MAX_UOP_ID \ No newline at end of file +PYSTATS_MAX_UOP_ID