From 7f50a5febd7af7259237a78dc533e9f9f274d51c Mon Sep 17 00:00:00 2001 From: Guo Ci Date: Mon, 12 Jan 2026 06:34:18 -0500 Subject: [PATCH 1/9] gh-140806: add docs for `enum.bin` function (#140807) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- Doc/library/enum.rst | 20 +++++++++++++++++++ Doc/library/functions.rst | 2 ++ Lib/enum.py | 3 ++- ...-10-30-19-28-42.gh-issue-140806.RBT9YH.rst | 1 + 4 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Documentation/2025-10-30-19-28-42.gh-issue-140806.RBT9YH.rst diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 0da27ba8e78284..b39164e54753a7 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -153,6 +153,12 @@ Module Contents Return a list of all power-of-two integers contained in a flag. + :func:`enum.bin` + + Like built-in :func:`bin`, except negative values are represented in + two's complement, and the leading bit always indicates sign + (``0`` implies positive, ``1`` implies negative). + .. versionadded:: 3.6 ``Flag``, ``IntFlag``, ``auto`` .. versionadded:: 3.11 ``StrEnum``, ``EnumCheck``, ``ReprEnum``, ``FlagBoundary``, ``property``, ``member``, ``nonmember``, ``global_enum``, ``show_flag_values`` @@ -1035,6 +1041,20 @@ Utilities and Decorators .. versionadded:: 3.11 +.. function:: bin(num, max_bits=None) + + Like built-in :func:`bin`, except negative values are represented in + two's complement, and the leading bit always indicates sign + (``0`` implies positive, ``1`` implies negative). + + >>> import enum + >>> enum.bin(10) + '0b0 1010' + >>> enum.bin(~10) # ~10 is -11 + '0b1 0101' + + .. versionadded:: 3.10 + --------------- Notes diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 7635e65296537d..cd819b8d06480a 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -138,6 +138,8 @@ are always available. They are listed here in alphabetical order. >>> f'{14:#b}', f'{14:b}' ('0b1110', '1110') + See also :func:`enum.bin` to represent negative values as twos-complement. + See also :func:`format` for more information. diff --git a/Lib/enum.py b/Lib/enum.py index 15dddf6de69268..025e973446d88d 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -130,7 +130,7 @@ def show_flag_values(value): def bin(num, max_bits=None): """ Like built-in bin(), except negative values are represented in - twos-compliment, and the leading bit always indicates sign + twos-complement, and the leading bit always indicates sign (0=positive, 1=negative). >>> bin(10) @@ -139,6 +139,7 @@ def bin(num, max_bits=None): '0b1 0101' """ + num = num.__index__() ceiling = 2 ** (num).bit_length() if num >= 0: s = bltns.bin(num + ceiling).replace('1', '0', 1) diff --git a/Misc/NEWS.d/next/Documentation/2025-10-30-19-28-42.gh-issue-140806.RBT9YH.rst b/Misc/NEWS.d/next/Documentation/2025-10-30-19-28-42.gh-issue-140806.RBT9YH.rst new file mode 100644 index 00000000000000..82bdf05d7300fa --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2025-10-30-19-28-42.gh-issue-140806.RBT9YH.rst @@ -0,0 +1 @@ +Add documentation for :func:`enum.bin`. From 42f7c2dfba58a8a8f31aba727d0fc51dd3ce2fce Mon Sep 17 00:00:00 2001 From: Yashraj Date: Mon, 12 Jan 2026 18:29:59 +0530 Subject: [PATCH 2/9] gh-141004: Document PyUnicode_IS_COMPACT and PyUnicode_IS_COMPACT_ASCII macros (GH-143494) --- Doc/c-api/unicode.rst | 21 +++++++++++++++++++++ Tools/check-c-api-docs/ignored_c_api.txt | 5 +---- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index ca7c8bb11a5d78..d2b6643c700e88 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -65,6 +65,27 @@ Python: .. versionadded:: 3.3 + The structure of a particular object can be determined using the following + macros. + The macros cannot fail; their behavior is undefined if their argument + is not a Python Unicode object. + + .. c:namespace:: NULL + + .. c:macro:: PyUnicode_IS_COMPACT(o) + + True if *o* uses the :c:struct:`PyCompactUnicodeObject` structure. + + .. versionadded:: 3.3 + + + .. c:macro:: PyUnicode_IS_COMPACT_ASCII(o) + + True if *o* uses the :c:struct:`PyASCIIObject` structure. + + .. versionadded:: 3.3 + + The following APIs are C macros and static inlined functions for fast checks and access to internal read-only data of Unicode objects: diff --git a/Tools/check-c-api-docs/ignored_c_api.txt b/Tools/check-c-api-docs/ignored_c_api.txt index 31c920555992bb..ebc0b5a8710ab5 100644 --- a/Tools/check-c-api-docs/ignored_c_api.txt +++ b/Tools/check-c-api-docs/ignored_c_api.txt @@ -101,9 +101,6 @@ PyUnstable_EXECUTABLE_KIND_PY_FUNCTION PyUnstable_EXECUTABLE_KIND_SKIP # cpython/pylifecycle.h Py_FrozenMain -# cpython/unicodeobject.h -PyUnicode_IS_COMPACT -PyUnicode_IS_COMPACT_ASCII # pythonrun.h PyErr_Display # cpython/objimpl.h @@ -139,4 +136,4 @@ PY_MONITORING_EVENT_BRANCH PY_DEF_EVENT PY_FOREACH_DICT_EVENT # cpython/pystats.h -PYSTATS_MAX_UOP_ID +PYSTATS_MAX_UOP_ID \ No newline at end of file From 054a565c64e486fbb35327690190b56750bb600b Mon Sep 17 00:00:00 2001 From: Cajetan Rodrigues Date: Mon, 12 Jan 2026 15:13:55 +0100 Subject: [PATCH 3/9] gh-134584: JIT: Remove redundant refcount for _BINARY_OP_SUBSCR_DICT (GH-143724) --- Include/internal/pycore_opcode_metadata.h | 4 +-- Include/internal/pycore_uop_ids.h | 2 +- Include/internal/pycore_uop_metadata.h | 8 ++--- Lib/test/test_capi/test_opt.py | 17 ++++++++++ Python/bytecodes.c | 12 ++++--- Python/executor_cases.c.h | 26 ++++++---------- Python/generated_cases.c.h | 38 ++++++++++++++--------- Python/optimizer_bytecodes.c | 6 ++++ Python/optimizer_cases.c.h | 14 +++++++-- 9 files changed, 83 insertions(+), 44 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index e225ccbf6ee7d4..0fd305dd72c322 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1098,7 +1098,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, - [BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [BINARY_OP_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_LIST_SLICE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1351,7 +1351,7 @@ _PyOpcode_macro_expansion[256] = { [BINARY_OP_INPLACE_ADD_UNICODE] = { .nuops = 3, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_INPLACE_ADD_UNICODE, OPARG_SIMPLE, 5 } } }, [BINARY_OP_MULTIPLY_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_MULTIPLY_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 } } }, [BINARY_OP_MULTIPLY_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_MULTIPLY_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_DICT] = { .nuops = 2, .uops = { { _GUARD_NOS_DICT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_DICT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_DICT] = { .nuops = 4, .uops = { { _GUARD_NOS_DICT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_DICT, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, [BINARY_OP_SUBSCR_GETITEM] = { .nuops = 4, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 5 }, { _BINARY_OP_SUBSCR_CHECK_FUNC, OPARG_SIMPLE, 5 }, { _BINARY_OP_SUBSCR_INIT_CALL, OPARG_SIMPLE, 5 }, { _PUSH_FRAME, OPARG_SIMPLE, 5 } } }, [BINARY_OP_SUBSCR_LIST_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_LIST_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, [BINARY_OP_SUBSCR_LIST_SLICE] = { .nuops = 3, .uops = { { _GUARD_TOS_SLICE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_LIST_SLICE, OPARG_SIMPLE, 5 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index b9297f37602a77..c11e72ab8c5cf7 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -388,7 +388,7 @@ extern "C" { #define _BINARY_OP_MULTIPLY_INT_r13 584 #define _BINARY_OP_MULTIPLY_INT_r23 585 #define _BINARY_OP_SUBSCR_CHECK_FUNC_r23 586 -#define _BINARY_OP_SUBSCR_DICT_r21 587 +#define _BINARY_OP_SUBSCR_DICT_r23 587 #define _BINARY_OP_SUBSCR_INIT_CALL_r01 588 #define _BINARY_OP_SUBSCR_INIT_CALL_r11 589 #define _BINARY_OP_SUBSCR_INIT_CALL_r21 590 diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index bce5bda8ff066c..4cc1184001089d 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -128,7 +128,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_SUBSCR_TUPLE_INT] = 0, [_GUARD_NOS_DICT] = HAS_EXIT_FLAG, [_GUARD_TOS_DICT] = HAS_EXIT_FLAG, - [_BINARY_OP_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_CHECK_FUNC] = HAS_DEOPT_FLAG, [_BINARY_OP_SUBSCR_INIT_CALL] = 0, [_LIST_APPEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG, @@ -1218,7 +1218,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 2, _BINARY_OP_SUBSCR_DICT_r21 }, + { 3, 2, _BINARY_OP_SUBSCR_DICT_r23 }, { -1, -1, -1 }, }, }, @@ -3594,7 +3594,7 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GUARD_TOS_DICT_r11] = _GUARD_TOS_DICT, [_GUARD_TOS_DICT_r22] = _GUARD_TOS_DICT, [_GUARD_TOS_DICT_r33] = _GUARD_TOS_DICT, - [_BINARY_OP_SUBSCR_DICT_r21] = _BINARY_OP_SUBSCR_DICT, + [_BINARY_OP_SUBSCR_DICT_r23] = _BINARY_OP_SUBSCR_DICT, [_BINARY_OP_SUBSCR_CHECK_FUNC_r23] = _BINARY_OP_SUBSCR_CHECK_FUNC, [_BINARY_OP_SUBSCR_INIT_CALL_r01] = _BINARY_OP_SUBSCR_INIT_CALL, [_BINARY_OP_SUBSCR_INIT_CALL_r11] = _BINARY_OP_SUBSCR_INIT_CALL, @@ -4107,7 +4107,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_BINARY_OP_SUBSCR_CHECK_FUNC] = "_BINARY_OP_SUBSCR_CHECK_FUNC", [_BINARY_OP_SUBSCR_CHECK_FUNC_r23] = "_BINARY_OP_SUBSCR_CHECK_FUNC_r23", [_BINARY_OP_SUBSCR_DICT] = "_BINARY_OP_SUBSCR_DICT", - [_BINARY_OP_SUBSCR_DICT_r21] = "_BINARY_OP_SUBSCR_DICT_r21", + [_BINARY_OP_SUBSCR_DICT_r23] = "_BINARY_OP_SUBSCR_DICT_r23", [_BINARY_OP_SUBSCR_INIT_CALL] = "_BINARY_OP_SUBSCR_INIT_CALL", [_BINARY_OP_SUBSCR_INIT_CALL_r01] = "_BINARY_OP_SUBSCR_INIT_CALL_r01", [_BINARY_OP_SUBSCR_INIT_CALL_r11] = "_BINARY_OP_SUBSCR_INIT_CALL_r11", diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index f111e9b5f2025b..79c5c22c13f557 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1975,6 +1975,23 @@ def testfunc(n): self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) self.assertLessEqual(count_ops(ex, "_POP_TOP_INT"), 1) + def test_binary_op_subscr_dict(self): + def testfunc(n): + x = 0 + d = {'a': 1, 'b': 2} + for _ in range(n): + v = d['a'] # _BINARY_OP_SUBSCR_DICT + if v == 1: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_SUBSCR_DICT", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) + def test_call_type_1_guards_removed(self): def testfunc(n): x = 0 diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0156fb3d06d854..18d66ae812fee9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1056,9 +1056,9 @@ dummy_func( } macro(BINARY_OP_SUBSCR_DICT) = - _GUARD_NOS_DICT + unused/5 + _BINARY_OP_SUBSCR_DICT; + _GUARD_NOS_DICT + unused/5 + _BINARY_OP_SUBSCR_DICT + POP_TOP + POP_TOP; - op(_BINARY_OP_SUBSCR_DICT, (dict_st, sub_st -- res)) { + op(_BINARY_OP_SUBSCR_DICT, (dict_st, sub_st -- res, ds, ss)) { PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); @@ -1069,9 +1069,13 @@ dummy_func( if (rc == 0) { _PyErr_SetKeyError(sub); } - DECREF_INPUTS(); - ERROR_IF(rc <= 0); // not found or error + if (rc <= 0) { + ERROR_NO_POP(); + } res = PyStackRef_FromPyObjectSteal(res_o); + ds = dict_st; + ss = sub_st; + INPUTS_DEAD(); } op(_BINARY_OP_SUBSCR_CHECK_FUNC, (container, unused -- container, unused, getitem)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index a4d4c4882118e3..1e1badd9a1f777 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -6034,12 +6034,14 @@ break; } - case _BINARY_OP_SUBSCR_DICT_r21: { + case _BINARY_OP_SUBSCR_DICT_r23: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef sub_st; _PyStackRef dict_st; _PyStackRef res; + _PyStackRef ds; + _PyStackRef ss; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; sub_st = _stack_item_1; @@ -6061,27 +6063,19 @@ _PyErr_SetKeyError(sub); stack_pointer = _PyFrame_GetStackPointer(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = sub_st; - sub_st = PyStackRef_NULL; - stack_pointer[-1] = sub_st; - PyStackRef_CLOSE(tmp); - tmp = dict_st; - dict_st = PyStackRef_NULL; - stack_pointer[-2] = dict_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (rc <= 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); + ds = dict_st; + ss = sub_st; + _tos_cache2 = ss; + _tos_cache1 = ds; _tos_cache0 = res; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e5e6d30f9c22f0..2efdf5b0b990df 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -611,6 +611,9 @@ _PyStackRef dict_st; _PyStackRef sub_st; _PyStackRef res; + _PyStackRef ds; + _PyStackRef ss; + _PyStackRef value; // _GUARD_NOS_DICT { nos = stack_pointer[-2]; @@ -639,26 +642,31 @@ _PyErr_SetKeyError(sub); stack_pointer = _PyFrame_GetStackPointer(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = sub_st; - sub_st = PyStackRef_NULL; - stack_pointer[-1] = sub_st; - PyStackRef_CLOSE(tmp); - tmp = dict_st; - dict_st = PyStackRef_NULL; - stack_pointer[-2] = dict_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (rc <= 0) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); + ds = dict_st; + ss = sub_st; + } + // _POP_TOP + { + value = ss; + stack_pointer[-2] = res; + stack_pointer[-1] = ds; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = ds; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); } - stack_pointer[0] = res; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index b50e68338c386e..e85536bfc3a493 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -380,6 +380,12 @@ dummy_func(void) { ss = sub_st; } + op(_BINARY_OP_SUBSCR_DICT, (dict_st, sub_st -- res, ds, ss)) { + res = sym_new_not_null(ctx); + ds = dict_st; + ss = sub_st; + } + op(_TO_BOOL, (value -- res)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &res, false); if (!already_bool) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 0a3f04fe0ab88f..898930a01b16af 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1072,11 +1072,21 @@ } case _BINARY_OP_SUBSCR_DICT: { + JitOptRef sub_st; + JitOptRef dict_st; JitOptRef res; + JitOptRef ds; + JitOptRef ss; + sub_st = stack_pointer[-1]; + dict_st = stack_pointer[-2]; res = sym_new_not_null(ctx); - CHECK_STACK_BOUNDS(-1); + ds = dict_st; + ss = sub_st; + CHECK_STACK_BOUNDS(1); stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[-1] = ds; + stack_pointer[0] = ss; + stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } From 971f387bbb598a481aa8974ddc7a358459030415 Mon Sep 17 00:00:00 2001 From: Kuang Yu Heng Date: Mon, 12 Jan 2026 07:30:35 -0800 Subject: [PATCH 4/9] gh-137113 docs: note readline no longer supported in REPL after 3.13 (GH-137142) Add a note to the readline module documentation stating that Python 3.13 and later no longer supports readline in the default REPL, as per gh-118840. Includes workaround using PYTHON_BASIC_REPL. Update tutorial to remove the reference, and use a different key to test things out. Signed-off-by: Kuang Yu Heng --- Doc/library/readline.rst | 6 ++++++ Doc/tutorial/interpreter.rst | 12 ++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 0449682585c8d0..199e17595f41ac 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -403,3 +403,9 @@ support history save/restore. :: def save_history(self, histfile): readline.set_history_length(1000) readline.write_history_file(histfile) + +.. note:: + + The new :term:`REPL` introduced in version 3.13 doesn't support readline. + However, readline can still be used by setting the :envvar:`PYTHON_BASIC_REPL` + environment variable. diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index cd52607142485e..72cac1c1e909d3 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -34,13 +34,13 @@ status. If that doesn't work, you can exit the interpreter by typing the following command: ``quit()``. The interpreter's line-editing features include interactive editing, history -substitution and code completion on systems that support the `GNU Readline -`_ library. +substitution and code completion on most systems. Perhaps the quickest check to see whether command line editing is supported is -typing :kbd:`Control-P` to the first Python prompt you get. If it beeps, you -have command line editing; see Appendix :ref:`tut-interacting` for an -introduction to the keys. If nothing appears to happen, or if ``^P`` is -echoed, command line editing isn't available; you'll only be able to use +typing a word in on the Python prompt, then pressing Left arrow (or :kbd:`Control-b`). +If the cursor moves, you have command line editing; see Appendix +:ref:`tut-interacting` for an introduction to the keys. +If nothing appears to happen, or if a sequence like ``^[[D`` or ``^B`` appears, +command line editing isn't available; you'll only be able to use backspace to remove characters from the current line. The interpreter operates somewhat like the Unix shell: when called with standard From fe78c1e749169946b43e93e0c605369cadb0c8f1 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 12 Jan 2026 15:37:43 +0000 Subject: [PATCH 5/9] gh-143253: Add libabigail suppression file for internal types (#143254) Co-authored-by: Petr Viktorin --- .github/CODEOWNERS | 3 +++ Makefile.pre.in | 2 +- Misc/libabigail.abignore | 25 +++++++++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 Misc/libabigail.abignore diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4789cd2c59e6cb..79091e2d4f2f4f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -143,6 +143,9 @@ Misc/externals.spdx.json @sethmlarson Misc/sbom.spdx.json @sethmlarson Tools/build/generate_sbom.py @sethmlarson +# ABI check +Misc/libabigail.abignore @encukou + # ---------------------------------------------------------------------------- # Platform Support diff --git a/Makefile.pre.in b/Makefile.pre.in index a6beb96d12a3f2..0b5aef5ee7e671 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1883,7 +1883,7 @@ regen-abidump: all .PHONY: check-abidump check-abidump: all - abidiff $(srcdir)/Doc/data/python$(LDVERSION).abi "libpython$(LDVERSION).so" --drop-private-types --no-architecture --no-added-syms + abidiff $(srcdir)/Doc/data/python$(LDVERSION).abi "libpython$(LDVERSION).so" --drop-private-types --no-architecture --no-added-syms --suppressions $(srcdir)/Misc/libabigail.abignore .PHONY: regen-limited-abi regen-limited-abi: all diff --git a/Misc/libabigail.abignore b/Misc/libabigail.abignore new file mode 100644 index 00000000000000..369813021faa3d --- /dev/null +++ b/Misc/libabigail.abignore @@ -0,0 +1,25 @@ +# libabigail suppression file for CPython ABI checks +# +# Suppress types defined directly in internal headers (pycore_*.h) +# Regex matches filenames NOT starting with "pycore_", so pycore_* types are suppressed. +[suppress_type] + source_location_not_regexp = ^([^p]|p[^y]|py[^c]|pyc[^o]|pyco[^r]|pycor[^e]|pycore[^_]) + accessed_through = pointer + +# Suppress public typedefs that alias internal structs. +# These are public names but their underlying struct layout is internal. +[suppress_type] + name = PyInterpreterState + accessed_through = pointer + +[suppress_type] + name = _PyRuntimeState + accessed_through = pointer + +[suppress_type] + name = PyThreadState + accessed_through = pointer + +[suppress_variable] + name = _PyRuntime + type_name = _PyRuntimeState From c3157480601499565fd42a8afbdb0207328ac484 Mon Sep 17 00:00:00 2001 From: VanshAgarwal24036 <148854295+VanshAgarwal24036@users.noreply.github.com> Date: Mon, 12 Jan 2026 21:09:32 +0530 Subject: [PATCH 6/9] gh-143544: Fix possible use-after-free in the JSON decoder when JSONDecodeError disappears during raising it (#143561) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Modules/_json.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/_json.c b/Modules/_json.c index 14714d4b346546..78a85496575a2c 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -423,11 +423,12 @@ raise_errmsg(const char *msg, PyObject *s, Py_ssize_t end) PyObject *exc; exc = PyObject_CallFunction(JSONDecodeError, "zOn", msg, s, end); - Py_DECREF(JSONDecodeError); if (exc) { PyErr_SetObject(JSONDecodeError, exc); Py_DECREF(exc); } + + Py_DECREF(JSONDecodeError); } static void From e535bdb0a2a9d1a0f588f23923ef3067fecdaeb6 Mon Sep 17 00:00:00 2001 From: Nadeshiko Manju Date: Mon, 12 Jan 2026 23:47:31 +0800 Subject: [PATCH 7/9] gh-134584: Eliminate redundant refcounting from `_CONTAINS_{OP|OP_SET|OP_DICT}` (GH-143731) Signed-off-by: Manjusaka --- Include/internal/pycore_opcode_metadata.h | 12 +- Include/internal/pycore_uop_ids.h | 6 +- Include/internal/pycore_uop_metadata.h | 24 ++-- Lib/test/test_capi/test_opt.py | 51 +++++++++ ...-01-12-22-49-36.gh-issue-134584.guDlsj.rst | 2 + Python/bytecodes.c | 36 ++++-- Python/executor_cases.c.h | 72 +++++------- Python/generated_cases.c.h | 108 +++++++++++------- Python/optimizer_bytecodes.c | 14 ++- Python/optimizer_cases.c.h | 58 ++++++++-- 10 files changed, 253 insertions(+), 130 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-01-12-22-49-36.gh-issue-134584.guDlsj.rst diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 0fd305dd72c322..98a6b991128738 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1154,9 +1154,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, - [CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CONTAINS_OP_SET] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CONTAINS_OP_SET] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1402,9 +1402,9 @@ _PyOpcode_macro_expansion[256] = { [COMPARE_OP_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _COMPARE_OP_FLOAT, OPARG_SIMPLE, 1 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 1 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 1 } } }, [COMPARE_OP_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _COMPARE_OP_INT, OPARG_SIMPLE, 1 }, { _POP_TOP_INT, OPARG_SIMPLE, 1 }, { _POP_TOP_INT, OPARG_SIMPLE, 1 } } }, [COMPARE_OP_STR] = { .nuops = 5, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _COMPARE_OP_STR, OPARG_SIMPLE, 1 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 1 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 1 } } }, - [CONTAINS_OP] = { .nuops = 1, .uops = { { _CONTAINS_OP, OPARG_SIMPLE, 0 } } }, - [CONTAINS_OP_DICT] = { .nuops = 2, .uops = { { _GUARD_TOS_DICT, OPARG_SIMPLE, 0 }, { _CONTAINS_OP_DICT, OPARG_SIMPLE, 1 } } }, - [CONTAINS_OP_SET] = { .nuops = 2, .uops = { { _GUARD_TOS_ANY_SET, OPARG_SIMPLE, 0 }, { _CONTAINS_OP_SET, OPARG_SIMPLE, 1 } } }, + [CONTAINS_OP] = { .nuops = 3, .uops = { { _CONTAINS_OP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [CONTAINS_OP_DICT] = { .nuops = 4, .uops = { { _GUARD_TOS_DICT, OPARG_SIMPLE, 0 }, { _CONTAINS_OP_DICT, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 } } }, + [CONTAINS_OP_SET] = { .nuops = 4, .uops = { { _GUARD_TOS_ANY_SET, OPARG_SIMPLE, 0 }, { _CONTAINS_OP_SET, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 } } }, [CONVERT_VALUE] = { .nuops = 1, .uops = { { _CONVERT_VALUE, OPARG_SIMPLE, 0 } } }, [COPY] = { .nuops = 1, .uops = { { _COPY, OPARG_SIMPLE, 0 } } }, [COPY_FREE_VARS] = { .nuops = 1, .uops = { { _COPY_FREE_VARS, OPARG_SIMPLE, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index c11e72ab8c5cf7..6af2acd5128d64 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -503,9 +503,9 @@ extern "C" { #define _COMPARE_OP_FLOAT_r23 699 #define _COMPARE_OP_INT_r23 700 #define _COMPARE_OP_STR_r23 701 -#define _CONTAINS_OP_r21 702 -#define _CONTAINS_OP_DICT_r21 703 -#define _CONTAINS_OP_SET_r21 704 +#define _CONTAINS_OP_r23 702 +#define _CONTAINS_OP_DICT_r23 703 +#define _CONTAINS_OP_SET_r23 704 #define _CONVERT_VALUE_r11 705 #define _COPY_r01 706 #define _COPY_1_r02 707 diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 4cc1184001089d..a14c1ccf4843ef 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -208,10 +208,10 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_COMPARE_OP_INT] = HAS_ARG_FLAG, [_COMPARE_OP_STR] = HAS_ARG_FLAG, [_IS_OP] = HAS_ARG_FLAG, - [_CONTAINS_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CONTAINS_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_TOS_ANY_SET] = HAS_DEOPT_FLAG, - [_CONTAINS_OP_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CONTAINS_OP_DICT] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CONTAINS_OP_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CONTAINS_OP_DICT] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CHECK_EG_MATCH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_EXC_MATCH] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_IMPORT_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -1938,7 +1938,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 2, _CONTAINS_OP_r21 }, + { 3, 2, _CONTAINS_OP_r23 }, { -1, -1, -1 }, }, }, @@ -1956,7 +1956,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 2, _CONTAINS_OP_SET_r21 }, + { 3, 2, _CONTAINS_OP_SET_r23 }, { -1, -1, -1 }, }, }, @@ -1965,7 +1965,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 2, _CONTAINS_OP_DICT_r21 }, + { 3, 2, _CONTAINS_OP_DICT_r23 }, { -1, -1, -1 }, }, }, @@ -3710,13 +3710,13 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_IS_OP_r03] = _IS_OP, [_IS_OP_r13] = _IS_OP, [_IS_OP_r23] = _IS_OP, - [_CONTAINS_OP_r21] = _CONTAINS_OP, + [_CONTAINS_OP_r23] = _CONTAINS_OP, [_GUARD_TOS_ANY_SET_r01] = _GUARD_TOS_ANY_SET, [_GUARD_TOS_ANY_SET_r11] = _GUARD_TOS_ANY_SET, [_GUARD_TOS_ANY_SET_r22] = _GUARD_TOS_ANY_SET, [_GUARD_TOS_ANY_SET_r33] = _GUARD_TOS_ANY_SET, - [_CONTAINS_OP_SET_r21] = _CONTAINS_OP_SET, - [_CONTAINS_OP_DICT_r21] = _CONTAINS_OP_DICT, + [_CONTAINS_OP_SET_r23] = _CONTAINS_OP_SET, + [_CONTAINS_OP_DICT_r23] = _CONTAINS_OP_DICT, [_CHECK_EG_MATCH_r22] = _CHECK_EG_MATCH, [_CHECK_EXC_MATCH_r22] = _CHECK_EXC_MATCH, [_IMPORT_NAME_r21] = _IMPORT_NAME, @@ -4288,11 +4288,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_COMPARE_OP_STR] = "_COMPARE_OP_STR", [_COMPARE_OP_STR_r23] = "_COMPARE_OP_STR_r23", [_CONTAINS_OP] = "_CONTAINS_OP", - [_CONTAINS_OP_r21] = "_CONTAINS_OP_r21", + [_CONTAINS_OP_r23] = "_CONTAINS_OP_r23", [_CONTAINS_OP_DICT] = "_CONTAINS_OP_DICT", - [_CONTAINS_OP_DICT_r21] = "_CONTAINS_OP_DICT_r21", + [_CONTAINS_OP_DICT_r23] = "_CONTAINS_OP_DICT_r23", [_CONTAINS_OP_SET] = "_CONTAINS_OP_SET", - [_CONTAINS_OP_SET_r21] = "_CONTAINS_OP_SET_r21", + [_CONTAINS_OP_SET_r23] = "_CONTAINS_OP_SET_r23", [_CONVERT_VALUE] = "_CONVERT_VALUE", [_CONVERT_VALUE_r11] = "_CONVERT_VALUE_r11", [_COPY] = "_COPY", diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 79c5c22c13f557..62c16fd6cb1db8 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1992,6 +1992,57 @@ def testfunc(n): self.assertIn("_BINARY_OP_SUBSCR_DICT", uops) self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) + def test_contains_op(self): + def testfunc(n): + x = 0 + items = [1, 2, 3] + for _ in range(n): + if 2 in items: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CONTAINS_OP", uops) + self.assertIn("_POP_TOP_NOP", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) + + def test_contains_op_set(self): + def testfunc(n): + x = 0 + s = {1, 2, 3} + for _ in range(n): + if 2 in s: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CONTAINS_OP_SET", uops) + self.assertIn("_POP_TOP_NOP", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) + + def test_contains_op_dict(self): + def testfunc(n): + x = 0 + d = {'a': 1, 'b': 2} + for _ in range(n): + if 'a' in d: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CONTAINS_OP_DICT", uops) + self.assertIn("_POP_TOP_NOP", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) + def test_call_type_1_guards_removed(self): def testfunc(n): x = 0 diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-01-12-22-49-36.gh-issue-134584.guDlsj.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-01-12-22-49-36.gh-issue-134584.guDlsj.rst new file mode 100644 index 00000000000000..712ddd3793194b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-01-12-22-49-36.gh-issue-134584.guDlsj.rst @@ -0,0 +1,2 @@ +Eliminate redundant refcounting from ``_CONTAINS_OP``, ``_CONTAINS_OP_SET`` +and ``_CONTAINS_OP_DICT``. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 18d66ae812fee9..880472713a8344 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2855,14 +2855,18 @@ dummy_func( CONTAINS_OP_DICT, }; - op(_CONTAINS_OP, (left, right -- b)) { + op(_CONTAINS_OP, (left, right -- b, l, r)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); int res = PySequence_Contains(right_o, left_o); - DECREF_INPUTS(); - ERROR_IF(res < 0); + if (res < 0) { + ERROR_NO_POP(); + } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + INPUTS_DEAD(); } specializing op(_SPECIALIZE_CONTAINS_OP, (counter/1, left, right -- left, right)) { @@ -2877,16 +2881,16 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION_FT */ } - macro(CONTAINS_OP) = _SPECIALIZE_CONTAINS_OP + _CONTAINS_OP; + macro(CONTAINS_OP) = _SPECIALIZE_CONTAINS_OP + _CONTAINS_OP + POP_TOP + POP_TOP; op(_GUARD_TOS_ANY_SET, (tos -- tos)) { PyObject *o = PyStackRef_AsPyObjectBorrow(tos); DEOPT_IF(!PyAnySet_CheckExact(o)); } - macro(CONTAINS_OP_SET) = _GUARD_TOS_ANY_SET + unused/1 + _CONTAINS_OP_SET; + macro(CONTAINS_OP_SET) = _GUARD_TOS_ANY_SET + unused/1 + _CONTAINS_OP_SET + POP_TOP + POP_TOP; - op(_CONTAINS_OP_SET, (left, right -- b)) { + op(_CONTAINS_OP_SET, (left, right -- b, l, r)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); @@ -2894,23 +2898,31 @@ dummy_func( STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! int res = _PySet_Contains((PySetObject *)right_o, left_o); - DECREF_INPUTS(); - ERROR_IF(res < 0); + if (res < 0) { + ERROR_NO_POP(); + } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + INPUTS_DEAD(); } - macro(CONTAINS_OP_DICT) = _GUARD_TOS_DICT + unused/1 + _CONTAINS_OP_DICT; + macro(CONTAINS_OP_DICT) = _GUARD_TOS_DICT + unused/1 + _CONTAINS_OP_DICT + POP_TOP + POP_TOP; - op(_CONTAINS_OP_DICT, (left, right -- b)) { + op(_CONTAINS_OP_DICT, (left, right -- b, l, r)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyDict_CheckExact(right_o)); STAT_INC(CONTAINS_OP, hit); int res = PyDict_Contains(right_o, left_o); - DECREF_INPUTS(); - ERROR_IF(res < 0); + if (res < 0) { + ERROR_NO_POP(); + } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + INPUTS_DEAD(); } inst(CHECK_EG_MATCH, (exc_value_st, match_type_st -- rest, match)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 1e1badd9a1f777..c2b9d9c31997b0 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -10150,12 +10150,14 @@ break; } - case _CONTAINS_OP_r21: { + case _CONTAINS_OP_r23: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; _PyStackRef left; _PyStackRef b; + _PyStackRef l; + _PyStackRef r; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); @@ -10169,26 +10171,20 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PySequence_Contains(right_o, left_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; _tos_cache0 = b; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } @@ -10278,12 +10274,14 @@ break; } - case _CONTAINS_OP_SET_r21: { + case _CONTAINS_OP_SET_r23: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; _PyStackRef left; _PyStackRef b; + _PyStackRef l; + _PyStackRef r; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); @@ -10299,36 +10297,32 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PySet_Contains((PySetObject *)right_o, left_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; _tos_cache0 = b; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _CONTAINS_OP_DICT_r21: { + case _CONTAINS_OP_DICT_r23: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; _PyStackRef left; _PyStackRef b; + _PyStackRef l; + _PyStackRef r; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); @@ -10344,26 +10338,20 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; _tos_cache0 = b; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2efdf5b0b990df..23f02c8c74d71f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5029,6 +5029,9 @@ _PyStackRef right; _PyStackRef left; _PyStackRef b; + _PyStackRef l; + _PyStackRef r; + _PyStackRef value; // _SPECIALIZE_CONTAINS_OP { right = stack_pointer[-1]; @@ -5053,25 +5056,32 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PySequence_Contains(right_o, left_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_LABEL(error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + } + // _POP_TOP + { + value = r; + stack_pointer[-2] = b; + stack_pointer[-1] = l; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = l; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); } - stack_pointer[0] = b; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5090,6 +5100,9 @@ _PyStackRef left; _PyStackRef right; _PyStackRef b; + _PyStackRef l; + _PyStackRef r; + _PyStackRef value; // _GUARD_TOS_DICT { tos = stack_pointer[-1]; @@ -5111,25 +5124,32 @@ STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_LABEL(error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + } + // _POP_TOP + { + value = r; + stack_pointer[-2] = b; + stack_pointer[-1] = l; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = l; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); } - stack_pointer[0] = b; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5148,6 +5168,9 @@ _PyStackRef left; _PyStackRef right; _PyStackRef b; + _PyStackRef l; + _PyStackRef r; + _PyStackRef value; // _GUARD_TOS_ANY_SET { tos = stack_pointer[-1]; @@ -5169,25 +5192,32 @@ STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PySet_Contains((PySetObject *)right_o, left_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_LABEL(error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + } + // _POP_TOP + { + value = r; + stack_pointer[-2] = b; + stack_pointer[-1] = l; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = l; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); } - stack_pointer[0] = b; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index e85536bfc3a493..49fb9e7625aaf2 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -525,17 +525,23 @@ dummy_func(void) { r = right; } - op(_CONTAINS_OP, (left, right -- b)) { - REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, b); + op(_CONTAINS_OP, (left, right -- b, l, r)) { b = sym_new_type(ctx, &PyBool_Type); + l = left; + r = right; + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, b); } - op(_CONTAINS_OP_SET, (left, right -- b)) { + op(_CONTAINS_OP_SET, (left, right -- b, l, r)) { b = sym_new_type(ctx, &PyBool_Type); + l = left; + r = right; } - op(_CONTAINS_OP_DICT, (left, right -- b)) { + op(_CONTAINS_OP_DICT, (left, right -- b, l, r)) { b = sym_new_type(ctx, &PyBool_Type); + l = left; + r = right; } op(_LOAD_CONST, (-- value)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 898930a01b16af..d522c5a0c19459 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2271,8 +2271,13 @@ JitOptRef right; JitOptRef left; JitOptRef b; + JitOptRef l; + JitOptRef r; right = stack_pointer[-1]; left = stack_pointer[-2]; + b = sym_new_type(ctx, &PyBool_Type); + l = left; + r = right; if ( sym_is_safe_const(ctx, left) && sym_is_safe_const(ctx, right) @@ -2282,33 +2287,42 @@ _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); _PyStackRef b_stackref; + _PyStackRef l_stackref; + _PyStackRef r_stackref; /* Start of uop copied from bytecodes for constant evaluation */ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); int res = PySequence_Contains(right_o, left_o); if (res < 0) { - goto error; + JUMP_TO_LABEL(error); } b_stackref = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + l_stackref = left; + r_stackref = right; /* End of uop copied from bytecodes for constant evaluation */ + (void)l_stackref; + (void)r_stackref; b = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(b_stackref)); if (sym_is_const(ctx, b)) { PyObject *result = sym_get_const(ctx, b); if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } - CHECK_STACK_BOUNDS(-1); + CHECK_STACK_BOUNDS(1); stack_pointer[-2] = b; - stack_pointer += -1; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } - b = sym_new_type(ctx, &PyBool_Type); - CHECK_STACK_BOUNDS(-1); + CHECK_STACK_BOUNDS(1); stack_pointer[-2] = b; - stack_pointer += -1; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2325,21 +2339,41 @@ } case _CONTAINS_OP_SET: { + JitOptRef right; + JitOptRef left; JitOptRef b; + JitOptRef l; + JitOptRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; b = sym_new_type(ctx, &PyBool_Type); - CHECK_STACK_BOUNDS(-1); + l = left; + r = right; + CHECK_STACK_BOUNDS(1); stack_pointer[-2] = b; - stack_pointer += -1; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _CONTAINS_OP_DICT: { + JitOptRef right; + JitOptRef left; JitOptRef b; + JitOptRef l; + JitOptRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; b = sym_new_type(ctx, &PyBool_Type); - CHECK_STACK_BOUNDS(-1); + l = left; + r = right; + CHECK_STACK_BOUNDS(1); stack_pointer[-2] = b; - stack_pointer += -1; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } From 1de46715ec50c7ae0b8e8671287239771c121e68 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Mon, 12 Jan 2026 17:43:05 +0100 Subject: [PATCH 8/9] gh-142518: Document thread-safety guarantees of list operations (#142519) * Add everything to code blocks * Improve wording around atomicity; specify exact types * Better explain lock-free and atomicity --- Doc/library/stdtypes.rst | 103 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index d03001f1cade05..22bc1536c1a37b 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1441,6 +1441,109 @@ application). list appear empty for the duration, and raises :exc:`ValueError` if it can detect that the list has been mutated during a sort. +.. admonition:: Thread safety + + Reading a single element from a :class:`list` is + :term:`atomic `: + + .. code-block:: + :class: green + + lst[i] # list.__getitem__ + + The following methods traverse the list and use :term:`atomic ` + reads of each item to perform their function. That means that they may + return results affected by concurrent modifications: + + .. code-block:: + :class: maybe + + item in lst + lst.index(item) + lst.count(item) + + All of the above methods/operations are also lock-free. They do not block + concurrent modifications. Other operations that hold a lock will not block + these from observing intermediate states. + + All other operations from here on block using the per-object lock. + + Writing a single item via ``lst[i] = x`` is safe to call from multiple + threads and will not corrupt the list. + + The following operations return new objects and appear + :term:`atomic ` to other threads: + + .. code-block:: + :class: good + + lst1 + lst2 # concatenates two lists into a new list + x * lst # repeats lst x times into a new list + lst.copy() # returns a shallow copy of the list + + Methods that only operate on a single elements with no shifting required are + :term:`atomic `: + + .. code-block:: + :class: good + + lst.append(x) # append to the end of the list, no shifting required + lst.pop() # pop element from the end of the list, no shifting required + + The :meth:`~list.clear` method is also :term:`atomic `. + Other threads cannot observe elements being removed. + + The :meth:`~list.sort` method is not :term:`atomic `. + Other threads cannot observe intermediate states during sorting, but the + list appears empty for the duration of the sort. + + The following operations may allow lock-free operations to observe + intermediate states since they modify multiple elements in place: + + .. code-block:: + :class: maybe + + lst.insert(idx, item) # shifts elements + lst.pop(idx) # idx not at the end of the list, shifts elements + lst *= x # copies elements in place + + The :meth:`~list.remove` method may allow concurrent modifications since + element comparison may execute arbitrary Python code (via + :meth:`~object.__eq__`). + + :meth:`~list.extend` is safe to call from multiple threads. However, its + guarantees depend on the iterable passed to it. If it is a :class:`list`, a + :class:`tuple`, a :class:`set`, a :class:`frozenset`, a :class:`dict` or a + :ref:`dictionary view object ` (but not their subclasses), the + ``extend`` operation is safe from concurrent modifications to the iterable. + Otherwise, an iterator is created which can be concurrently modified by + another thread. The same applies to inplace concatenation of a list with + other iterables when using ``lst += iterable``. + + Similarly, assigning to a list slice with ``lst[i:j] = iterable`` is safe + to call from multiple threads, but ``iterable`` is only locked when it is + also a :class:`list` (but not its subclasses). + + Operations that involve multiple accesses, as well as iteration, are never + atomic. For example: + + .. code-block:: + :class: bad + + # NOT atomic: read-modify-write + lst[i] = lst[i] + 1 + + # NOT atomic: check-then-act + if lst: + item = lst.pop() + + # NOT thread-safe: iteration while modifying + for item in lst: + process(item) # another thread may modify lst + + Consider external synchronization when sharing :class:`list` instances + across threads. See :ref:`freethreading-python-howto` for more information. + .. _typesseq-tuple: From 7d155d7915dcb46f0b900010b11804c1c0da0719 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 12 Jan 2026 12:14:13 -0500 Subject: [PATCH 9/9] gh-140795: Remove 'exc' field in SSLObject (gh-143491) The 'exc' field was used by our debug SSL callbacks. Keep the exception in the normal per-thread state to avoid shared mutable state between threads. This also avoids a reference count leak if the Python callback raised an exception because it can be called multiple times per SSL operation. --- Lib/test/test_ssl.py | 14 +++++ Modules/_ssl.c | 117 +++++++++++++++++++++--------------- Modules/_ssl/debughelpers.c | 19 ++++-- 3 files changed, 97 insertions(+), 53 deletions(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index ebdf5455163c65..9dc99fbf5cf7d2 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -5277,6 +5277,20 @@ def msg_cb(conn, direction, version, content_type, msg_type, data): with self.assertRaises(TypeError): client_context._msg_callback = object() + def test_msg_callback_exception(self): + client_context, server_context, hostname = testing_context() + + def msg_cb(conn, direction, version, content_type, msg_type, data): + raise RuntimeError("msg_cb exception") + + client_context._msg_callback = msg_cb + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + with self.assertRaisesRegex(RuntimeError, "msg_cb exception"): + s.connect((HOST, server.port)) + def test_msg_callback_tls12(self): client_context, server_context, hostname = testing_context() client_context.maximum_version = ssl.TLSVersion.TLSv1_2 diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 5d2f075ed0c675..7dd57e7892af41 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -360,11 +360,6 @@ typedef struct { enum py_ssl_server_or_client socket_type; PyObject *owner; /* Python level "owner" passed to servername callback */ PyObject *server_hostname; - /* Some SSL callbacks don't have error reporting. Callback wrappers - * store exception information on the socket. The handshake, read, write, - * and shutdown methods check for chained exceptions. - */ - PyObject *exc; } PySSLSocket; #define PySSLSocket_CAST(op) ((PySSLSocket *)(op)) @@ -657,18 +652,12 @@ fill_and_set_sslerror(_sslmodulestate *state, PyUnicodeWriter_Discard(writer); } -static int -PySSL_ChainExceptions(PySSLSocket *sslsock) { - if (sslsock->exc == NULL) - return 0; - - _PyErr_ChainExceptions1(sslsock->exc); - sslsock->exc = NULL; - return -1; -} - +// Set the appropriate SSL error exception. +// err - error information from SSL and libc +// exc - if not NULL, an exception from _debughelpers.c callback to be chained static PyObject * -PySSL_SetError(PySSLSocket *sslsock, _PySSLError err, const char *filename, int lineno) +PySSL_SetError(PySSLSocket *sslsock, _PySSLError err, PyObject *exc, + const char *filename, int lineno) { PyObject *type; char *errstr = NULL; @@ -776,7 +765,7 @@ PySSL_SetError(PySSLSocket *sslsock, _PySSLError err, const char *filename, int } fill_and_set_sslerror(state, sslsock, type, p, errstr, lineno, e); ERR_clear_error(); - PySSL_ChainExceptions(sslsock); + _PyErr_ChainExceptions1(exc); // chain any exceptions from callbacks return NULL; } @@ -908,7 +897,6 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, self->shutdown_seen_zero = 0; self->owner = NULL; self->server_hostname = NULL; - self->exc = NULL; /* Make sure the SSL error state is initialized */ ERR_clear_error(); @@ -1029,6 +1017,7 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) { int ret; _PySSLError err; + PyObject *exc = NULL; int sockstate, nonblocking; PySocketSockObject *sock = GET_SOCKET(self); PyTime_t timeout, deadline = 0; @@ -1064,6 +1053,12 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) Py_END_ALLOW_THREADS; _PySSL_FIX_ERRNO; + // Get any exception that occurred in a debughelpers.c callback + exc = PyErr_GetRaisedException(); + if (exc != NULL) { + break; + } + if (PyErr_CheckSignals()) goto error; @@ -1098,13 +1093,15 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) Py_XDECREF(sock); if (ret < 1) - return PySSL_SetError(self, err, __FILE__, __LINE__); - if (PySSL_ChainExceptions(self) < 0) + return PySSL_SetError(self, err, exc, __FILE__, __LINE__); + if (exc != NULL) { + PyErr_SetRaisedException(exc); return NULL; + } Py_RETURN_NONE; error: + assert(exc == NULL); Py_XDECREF(sock); - PySSL_ChainExceptions(self); return NULL; } @@ -2434,17 +2431,7 @@ _ssl__SSLSocket_owner_set_impl(PySSLSocket *self, PyObject *value) static int PySSL_traverse(PyObject *op, visitproc visit, void *arg) { - PySSLSocket *self = PySSLSocket_CAST(op); - Py_VISIT(self->exc); - Py_VISIT(Py_TYPE(self)); - return 0; -} - -static int -PySSL_clear(PyObject *op) -{ - PySSLSocket *self = PySSLSocket_CAST(op); - Py_CLEAR(self->exc); + Py_VISIT(Py_TYPE(op)); return 0; } @@ -2619,6 +2606,7 @@ _ssl__SSLSocket_sendfile_impl(PySSLSocket *self, int fd, Py_off_t offset, Py_ssize_t retval; int sockstate; _PySSLError err; + PyObject *exc = NULL; PySocketSockObject *sock = GET_SOCKET(self); PyTime_t timeout, deadline = 0; int has_timeout; @@ -2666,6 +2654,11 @@ _ssl__SSLSocket_sendfile_impl(PySSLSocket *self, int fd, Py_off_t offset, Py_END_ALLOW_THREADS; _PySSL_FIX_ERRNO; + exc = PyErr_GetRaisedException(); + if (exc != NULL) { + break; + } + if (PyErr_CheckSignals()) { goto error; } @@ -2715,15 +2708,18 @@ _ssl__SSLSocket_sendfile_impl(PySSLSocket *self, int fd, Py_off_t offset, } Py_XDECREF(sock); if (retval < 0) { - return PySSL_SetError(self, err, __FILE__, __LINE__); + return PySSL_SetError(self, err, exc, __FILE__, __LINE__); } - if (PySSL_ChainExceptions(self) < 0) { + if (exc != NULL) { + PyErr_SetRaisedException(exc); return NULL; } return PyLong_FromSize_t(retval); error: Py_XDECREF(sock); - (void)PySSL_ChainExceptions(self); + if (exc != NULL) { + _PyErr_ChainExceptions1(exc); + } return NULL; } #endif /* BIO_get_ktls_send */ @@ -2747,6 +2743,7 @@ _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) int retval; int sockstate; _PySSLError err; + PyObject *exc = NULL; int nonblocking; PySocketSockObject *sock = GET_SOCKET(self); PyTime_t timeout, deadline = 0; @@ -2797,6 +2794,11 @@ _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) Py_END_ALLOW_THREADS; _PySSL_FIX_ERRNO; + exc = PyErr_GetRaisedException(); + if (exc != NULL) { + break; + } + if (PyErr_CheckSignals()) goto error; @@ -2828,13 +2830,15 @@ _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) Py_XDECREF(sock); if (retval == 0) - return PySSL_SetError(self, err, __FILE__, __LINE__); - if (PySSL_ChainExceptions(self) < 0) + return PySSL_SetError(self, err, exc, __FILE__, __LINE__); + if (exc != NULL) { + PyErr_SetRaisedException(exc); return NULL; + } return PyLong_FromSize_t(count); error: + assert(exc == NULL); Py_XDECREF(sock); - PySSL_ChainExceptions(self); return NULL; } @@ -2860,7 +2864,7 @@ _ssl__SSLSocket_pending_impl(PySSLSocket *self) _PySSL_FIX_ERRNO; if (count < 0) - return PySSL_SetError(self, err, __FILE__, __LINE__); + return PySSL_SetError(self, err, NULL, __FILE__, __LINE__); else return PyLong_FromLong(count); } @@ -2888,6 +2892,7 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, int retval; int sockstate; _PySSLError err; + PyObject *exc = NULL; int nonblocking; PySocketSockObject *sock = GET_SOCKET(self); PyTime_t timeout, deadline = 0; @@ -2955,6 +2960,11 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, Py_END_ALLOW_THREADS; _PySSL_FIX_ERRNO; + exc = PyErr_GetRaisedException(); + if (exc != NULL) { + break; + } + if (PyErr_CheckSignals()) goto error; @@ -2986,13 +2996,18 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, err.ssl == SSL_ERROR_WANT_WRITE); if (retval == 0) { - PySSL_SetError(self, err, __FILE__, __LINE__); + PySSL_SetError(self, err, exc, __FILE__, __LINE__); + exc = NULL; goto error; } - if (self->exc != NULL) + else if (exc != NULL) { + PyErr_SetRaisedException(exc); + exc = NULL; goto error; + } done: + assert(exc == NULL); Py_XDECREF(sock); if (!group_right_1) { return PyBytesWriter_FinishWithSize(writer, count); @@ -3002,7 +3017,7 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, } error: - PySSL_ChainExceptions(self); + assert(exc == NULL); Py_XDECREF(sock); if (!group_right_1) { PyBytesWriter_Discard(writer); @@ -3022,6 +3037,7 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) /*[clinic end generated code: output=ca1aa7ed9d25ca42 input=98d9635cd4e16514]*/ { _PySSLError err; + PyObject *exc = NULL; int sockstate, nonblocking, ret; int zeros = 0; PySocketSockObject *sock = GET_SOCKET(self); @@ -3067,6 +3083,11 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) Py_END_ALLOW_THREADS; _PySSL_FIX_ERRNO; + exc = PyErr_GetRaisedException(); + if (exc != NULL) { + break; + } + /* If err == 1, a secure shutdown with SSL_shutdown() is complete */ if (ret > 0) break; @@ -3113,11 +3134,14 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) } if (ret < 0) { Py_XDECREF(sock); - PySSL_SetError(self, err, __FILE__, __LINE__); + PySSL_SetError(self, err, exc, __FILE__, __LINE__); + return NULL; + } + else if (exc != NULL) { + Py_XDECREF(sock); + PyErr_SetRaisedException(exc); return NULL; } - if (self->exc != NULL) - goto error; if (sock) /* It's already INCREF'ed */ return (PyObject *) sock; @@ -3125,8 +3149,8 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) Py_RETURN_NONE; error: + assert(exc == NULL); Py_XDECREF(sock); - PySSL_ChainExceptions(self); return NULL; } @@ -3335,7 +3359,6 @@ static PyType_Slot PySSLSocket_slots[] = { {Py_tp_getset, ssl_getsetlist}, {Py_tp_dealloc, PySSL_dealloc}, {Py_tp_traverse, PySSL_traverse}, - {Py_tp_clear, PySSL_clear}, {0, 0}, }; diff --git a/Modules/_ssl/debughelpers.c b/Modules/_ssl/debughelpers.c index 866c172e4996f7..e0cb7ca9a09f91 100644 --- a/Modules/_ssl/debughelpers.c +++ b/Modules/_ssl/debughelpers.c @@ -26,6 +26,8 @@ _PySSL_msg_callback(int write_p, int version, int content_type, return; } + PyObject *exc = PyErr_GetRaisedException(); + PyObject *ssl_socket; /* ssl.SSLSocket or ssl.SSLObject */ if (ssl_obj->owner) PyWeakref_GetRef(ssl_obj->owner, &ssl_socket); @@ -73,13 +75,13 @@ _PySSL_msg_callback(int write_p, int version, int content_type, version, content_type, msg_type, buf, len ); - if (res == NULL) { - ssl_obj->exc = PyErr_GetRaisedException(); - } else { - Py_DECREF(res); - } + Py_XDECREF(res); Py_XDECREF(ssl_socket); + if (exc != NULL) { + _PyErr_ChainExceptions1(exc); + } + PyGILState_Release(threadstate); } @@ -122,10 +124,13 @@ _PySSL_keylog_callback(const SSL *ssl, const char *line) { PyGILState_STATE threadstate; PySSLSocket *ssl_obj = NULL; /* ssl._SSLSocket, borrowed ref */ + PyObject *exc; int res, e; threadstate = PyGILState_Ensure(); + exc = PyErr_GetRaisedException(); + ssl_obj = (PySSLSocket *)SSL_get_app_data(ssl); assert(Py_IS_TYPE(ssl_obj, get_state_sock(ssl_obj)->PySSLSocket_Type)); PyThread_type_lock lock = get_state_sock(ssl_obj)->keylog_lock; @@ -153,10 +158,12 @@ _PySSL_keylog_callback(const SSL *ssl, const char *line) errno = e; PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, ssl_obj->ctx->keylog_filename); - ssl_obj->exc = PyErr_GetRaisedException(); } done: + if (exc != NULL) { + _PyErr_ChainExceptions1(exc); + } PyGILState_Release(threadstate); }