[pull] main from python:main

.github/workflows/jit.yml

@@ -7,6 +7,7 @@ on:
       - 'Python/optimizer*.c'
       - 'Python/executor_cases.c.h'
       - 'Python/optimizer_cases.c.h'
+      - '**_testinternalcapi**'
       - '!Python/perf_jit_trampoline.c'
       - '!**/*.md'
       - '!**/*.ini'
@@ -17,6 +18,7 @@ on:
       - 'Python/optimizer*.c'
       - 'Python/executor_cases.c.h'
       - 'Python/optimizer_cases.c.h'
+      - '**_testinternalcapi**'
       - '!Python/perf_jit_trampoline.c'
       - '!**/*.md'
       - '!**/*.ini'

Doc/c-api/descriptor.rst

@@ -10,11 +10,6 @@ found in the dictionary of type objects.
 
 .. XXX document these!
 
-.. c:var:: PyTypeObject PyProperty_Type
-
-   The type object for the built-in descriptor types.
-
-
 .. c:function:: PyObject* PyDescr_NewGetSet(PyTypeObject *type, struct PyGetSetDef *getset)
 
 
@@ -74,9 +69,26 @@ found in the dictionary of type objects.
 .. c:function:: PyObject* PyWrapper_New(PyObject *, PyObject *)
 
 
+.. c:macro:: PyDescr_COMMON
+
+   This is a :term:`soft deprecated` macro including the common fields for a
+   descriptor object.
+
+   This was included in Python's C API by mistake; do not use it in extensions.
+   For creating custom descriptor objects, create a class implementing the
+   descriptor protocol (:c:member:`~PyTypeObject.tp_descr_get` and
+   :c:member:`~PyTypeObject.tp_descr_set`).
+
+
 Built-in descriptors
 ^^^^^^^^^^^^^^^^^^^^
 
+.. c:var:: PyTypeObject PyProperty_Type
+
+   The type object for property objects. This is the same object as
+   :class:`property` in the Python layer.
+
+
 .. c:var:: PyTypeObject PySuper_Type
 
    The type object for super objects. This is the same object as

Doc/c-api/exceptions.rst

@@ -793,6 +793,17 @@ Exception Classes
    Return :c:member:`~PyTypeObject.tp_name` of the exception class *ob*.
 
 
+.. c:macro:: PyException_HEAD
+
+   This is a :term:`soft deprecated` macro including the base fields for an
+   exception object.
+
+   This was included in Python's C API by mistake and is not designed for use
+   in extensions. For creating custom exception objects, use
+   :c:func:`PyErr_NewException` or otherwise create a class inheriting from
+   :c:data:`PyExc_BaseException`.
+
+
 Exception Objects
 =================
 

Include/internal/pycore_optimizer.h

@@ -115,8 +115,10 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst)
 
 
 #define REF_IS_BORROWED 1
+#define REF_IS_INVALID  2
+#define REF_TAG_BITS    3
 
-#define JIT_BITS_TO_PTR_MASKED(REF) ((JitOptSymbol *)(((REF).bits) & (~REF_IS_BORROWED)))
+#define JIT_BITS_TO_PTR_MASKED(REF) ((JitOptSymbol *)(((REF).bits) & (~REF_TAG_BITS)))
 
 static inline JitOptSymbol *
 PyJitRef_Unwrap(JitOptRef ref)
@@ -133,6 +135,18 @@ PyJitRef_Wrap(JitOptSymbol *sym)
     return (JitOptRef){.bits=(uintptr_t)sym};
 }
 
+static inline JitOptRef
+PyJitRef_WrapInvalid(void *ptr)
+{
+    return (JitOptRef){.bits=(uintptr_t)ptr | REF_IS_INVALID};
+}
+
+static inline bool
+PyJitRef_IsInvalid(JitOptRef ref)
+{
+    return (ref.bits & REF_IS_INVALID) == REF_IS_INVALID;
+}
+
 static inline JitOptRef
 PyJitRef_StripReferenceInfo(JitOptRef ref)
 {
@@ -225,15 +239,15 @@ PyAPI_FUNC(int) _PyDumpExecutors(FILE *out);
 PyAPI_FUNC(void) _Py_ClearExecutorDeletionList(PyInterpreterState *interp);
 #endif
 
-int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int stop_tracing_opcode);
+PyAPI_FUNC(int) _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int stop_tracing_opcode);
 
 PyAPI_FUNC(int)
 _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame,
     _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *start_instr,
     _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit,
     int oparg, _PyExecutorObject *current_executor);
 
-void _PyJit_FinalizeTracing(PyThreadState *tstate, int err);
+PyAPI_FUNC(void) _PyJit_FinalizeTracing(PyThreadState *tstate, int err);
 void _PyJit_TracerFree(_PyThreadStateImpl *_tstate);
 
 void _PyJit_Tracer_InvalidateDependency(PyThreadState *old_tstate, void *obj);

Include/internal/pycore_optimizer_types.h

@@ -8,6 +8,7 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include <stdbool.h>
 #include "pycore_uop.h"  // UOP_MAX_TRACE_LENGTH
 
 // Holds locals, stack, locals, stack ... (in that order)
@@ -128,6 +129,8 @@ typedef struct _JitOptContext {
     JitOptRef *n_consumed;
     JitOptRef *limit;
     JitOptRef locals_and_stack[MAX_ABSTRACT_INTERP_SIZE];
+    _PyUOpInstruction *out_buffer;
+    int out_len;
 } JitOptContext;
 
 

Include/internal/pycore_tstate.h

@@ -60,6 +60,7 @@ typedef struct _PyJitTracerState {
     _PyJitTracerTranslatorState translator_state;
     JitOptContext opt_context;
     _PyUOpInstruction code_buffer[UOP_MAX_TRACE_LENGTH];
+    _PyUOpInstruction out_buffer[UOP_MAX_TRACE_LENGTH];
 } _PyJitTracerState;
 
 #endif

Python/optimizer.c

@@ -1535,6 +1535,7 @@ uop_optimize(
         if (length <= 0) {
             return length;
         }
+        buffer = _tstate->jit_tracer_state->out_buffer;
     }
     assert(length < UOP_MAX_TRACE_LENGTH/2);
     assert(length >= 1);

Python/optimizer_analysis.c

@@ -39,6 +39,7 @@
 #ifdef Py_DEBUG
     extern const char *_PyUOpName(int index);
     extern void _PyUOpPrint(const _PyUOpInstruction *uop);
+    extern void _PyUOpSymPrint(JitOptRef ref);
     static const char *const DEBUG_ENV = "PYTHON_OPT_DEBUG";
     static inline int get_lltrace(void) {
         char *uop_debug = Py_GETENV(DEBUG_ENV);
@@ -50,6 +51,38 @@
     }
     #define DPRINTF(level, ...) \
     if (get_lltrace() >= (level)) { printf(__VA_ARGS__); }
+
+
+
+static void
+dump_abstract_stack(_Py_UOpsAbstractFrame *frame, JitOptRef *stack_pointer)
+{
+    JitOptRef *stack_base = frame->stack;
+    JitOptRef *locals_base = frame->locals;
+    printf("    locals=[");
+    for (JitOptRef *ptr = locals_base; ptr < stack_base; ptr++) {
+        if (ptr != locals_base) {
+            printf(", ");
+        }
+        _PyUOpSymPrint(*ptr);
+    }
+    printf("]\n");
+    if (stack_pointer < stack_base) {
+        printf("    stack=%d\n", (int)(stack_pointer - stack_base));
+    }
+    else {
+        printf("    stack=[");
+        for (JitOptRef *ptr = stack_base; ptr < stack_pointer; ptr++) {
+            if (ptr != stack_base) {
+                printf(", ");
+            }
+            _PyUOpSymPrint(*ptr);
+        }
+        printf("]\n");
+    }
+    fflush(stdout);
+}
+
 #else
     #define DPRINTF(level, ...)
 #endif
@@ -143,6 +176,18 @@ incorrect_keys(PyObject *obj, uint32_t version)
 #define STACK_LEVEL()     ((int)(stack_pointer - ctx->frame->stack))
 #define STACK_SIZE()      ((int)(ctx->frame->stack_len))
 
+static inline int
+is_terminator_uop(const _PyUOpInstruction *uop)
+{
+    int opcode = uop->opcode;
+    return (
+        opcode == _EXIT_TRACE ||
+        opcode == _JUMP_TO_TOP ||
+        opcode == _DYNAMIC_EXIT ||
+        opcode == _DEOPT
+    );
+}
+
 #define CURRENT_FRAME_IS_INIT_SHIM() (ctx->frame->code == ((PyCodeObject *)&_Py_InitCleanup))
 
 #define GETLOCAL(idx)          ((ctx->frame->locals[idx]))
@@ -152,6 +197,22 @@ incorrect_keys(PyObject *obj, uint32_t version)
     (INST)->oparg = ARG;            \
     (INST)->operand0 = OPERAND;
 
+#define ADD_OP(OP, ARG, OPERAND) add_op(ctx, this_instr, (OP), (ARG), (OPERAND))
+
+static inline void
+add_op(JitOptContext *ctx, _PyUOpInstruction *this_instr,
+       uint16_t opcode, uint16_t oparg, uintptr_t operand0)
+{
+    _PyUOpInstruction *out = &ctx->out_buffer[ctx->out_len];
+    out->opcode = (opcode);
+    out->format = this_instr->format;
+    out->oparg = (oparg);
+    out->target = this_instr->target;
+    out->operand0 = (operand0);
+    out->operand1 = this_instr->operand1;
+    ctx->out_len++;
+}
+
 /* Shortened forms for convenience, used in optimizer_bytecodes.c */
 #define sym_is_not_null _Py_uop_sym_is_not_null
 #define sym_is_const _Py_uop_sym_is_const
@@ -219,7 +280,7 @@ optimize_to_bool(
     bool insert_mode)
 {
     if (sym_matches_type(value, &PyBool_Type)) {
-        REPLACE_OP(this_instr, _NOP, 0, 0);
+        ADD_OP(_NOP, 0, 0);
         *result_ptr = value;
         return 1;
     }
@@ -229,17 +290,17 @@ optimize_to_bool(
         int opcode = insert_mode ?
             _INSERT_1_LOAD_CONST_INLINE_BORROW :
             _POP_TOP_LOAD_CONST_INLINE_BORROW;
-        REPLACE_OP(this_instr, opcode, 0, (uintptr_t)load);
+        ADD_OP(opcode, 0, (uintptr_t)load);
         *result_ptr = sym_new_const(ctx, load);
         return 1;
     }
     return 0;
 }
 
 static void
-eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit)
+eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, bool exit)
 {
-    REPLACE_OP(this_instr, _POP_TOP, 0, 0);
+    ADD_OP(_POP_TOP, 0, 0);
     if (exit) {
         REPLACE_OP((this_instr+1), _EXIT_TRACE, 0, 0);
         this_instr[1].target = this_instr->target;
@@ -256,7 +317,7 @@ lookup_attr(JitOptContext *ctx, _PyBloomFilter *dependencies, _PyUOpInstruction
         PyObject *lookup = _PyType_Lookup(type, name);
         if (lookup) {
             int opcode = _Py_IsImmortal(lookup) ? immortal : mortal;
-            REPLACE_OP(this_instr, opcode, 0, (uintptr_t)lookup);
+            ADD_OP(opcode, 0, (uintptr_t)lookup);
             PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type);
             _Py_BloomFilter_Add(dependencies, type);
             return sym_new_const(ctx, lookup);
@@ -349,6 +410,8 @@ optimize_uops(
     JitOptContext *ctx = &tstate->jit_tracer_state->opt_context;
     uint32_t opcode = UINT16_MAX;
 
+    ctx->out_buffer = tstate->jit_tracer_state->out_buffer;
+
     // Make sure that watchers are set up
     PyInterpreterState *interp = _PyInterpreterState_GET();
     if (interp->dict_state.watchers[GLOBALS_WATCHER_ID] == NULL) {
@@ -365,6 +428,8 @@ optimize_uops(
     ctx->curr_frame_depth++;
     ctx->frame = frame;
 
+    ctx->out_len = 0;
+
     _PyUOpInstruction *this_instr = NULL;
     JitOptRef *stack_pointer = ctx->frame->stack_pointer;
 
@@ -383,7 +448,10 @@ optimize_uops(
         if (get_lltrace() >= 3) {
             printf("%4d abs: ", (int)(this_instr - trace));
             _PyUOpPrint(this_instr);
-            printf(" ");
+            printf(" \n");
+            if (get_lltrace() >= 5 && !CURRENT_FRAME_IS_INIT_SHIM()) {
+                dump_abstract_stack(ctx->frame, stack_pointer);
+            }
         }
 #endif
 
@@ -395,6 +463,10 @@ optimize_uops(
                 DPRINTF(1, "\nUnknown opcode in abstract interpreter\n");
                 Py_UNREACHABLE();
         }
+        // If no ADD_OP was called during this iteration, copy the original instruction
+        if (ctx->out_len == i) {
+            ctx->out_buffer[ctx->out_len++] = *this_instr;
+        }
         assert(ctx->frame != NULL);
         if (!CURRENT_FRAME_IS_INIT_SHIM()) {
             DPRINTF(3, " stack_level %d\n", STACK_LEVEL());
@@ -423,7 +495,21 @@ optimize_uops(
     /* Either reached the end or cannot optimize further, but there
      * would be no benefit in retrying later */
     _Py_uop_abstractcontext_fini(ctx);
-    return trace_len;
+    // Check that the trace ends with a proper terminator
+    if (ctx->out_len > 0) {
+        _PyUOpInstruction *last_uop = &ctx->out_buffer[ctx->out_len - 1];
+        if (!is_terminator_uop(last_uop)) {
+            // Copy remaining uops from original trace until we find a terminator
+            for (int i = ctx->out_len; i < trace_len; i++) {
+                ctx->out_buffer[ctx->out_len++] = trace[i];
+                if (is_terminator_uop(&trace[i])) {
+                    break;
+                }
+            }
+        }
+    }
+
+    return ctx->out_len;
 
 error:
     DPRINTF(3, "\n");
@@ -595,7 +681,7 @@ _Py_uop_analyze_and_optimize(
 
     assert(length > 0);
 
-    length = remove_unneeded_uops(buffer, length);
+    length = remove_unneeded_uops(tstate->jit_tracer_state->out_buffer, length);
     assert(length > 0);
 
     OPT_STAT_INC(optimizer_successes);