diff options
| -rw-r--r-- | Include/internal/pycore_list.h | 8 | ||||
| -rw-r--r-- | Include/internal/pycore_opcode_metadata.h | 4 | ||||
| -rw-r--r-- | Include/internal/pycore_uop_ids.h | 153 | ||||
| -rw-r--r-- | Include/internal/pycore_uop_metadata.h | 6 | ||||
| -rw-r--r-- | Lib/test/test_opcache.py | 39 | ||||
| -rw-r--r-- | Objects/listobject.c | 26 | ||||
| -rw-r--r-- | Python/bytecodes.c | 113 | ||||
| -rw-r--r-- | Python/executor_cases.c.h | 79 | ||||
| -rw-r--r-- | Python/generated_cases.c.h | 105 | ||||
| -rw-r--r-- | Python/optimizer.c | 1 | ||||
| -rw-r--r-- | Python/optimizer_cases.c.h | 4 | ||||
| -rw-r--r-- | Python/specialize.c | 53 | ||||
| -rw-r--r-- | Tools/cases_generator/analyzer.py | 3 |
13 files changed, 469 insertions, 125 deletions
diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 5d817891408..5c66d76a0d6 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -8,11 +8,19 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#ifdef Py_GIL_DISABLED +#include "pycore_stackref.h" +#endif + PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *); extern void _PyList_DebugMallocStats(FILE *out); // _PyList_GetItemRef should be used only when the object is known as a list // because it doesn't raise TypeError when the object is not a list, whereas PyList_GetItemRef does. extern PyObject* _PyList_GetItemRef(PyListObject *, Py_ssize_t i); +#ifdef Py_GIL_DISABLED +// Returns -1 in case of races with other threads. +extern int _PyList_GetItemRefNoLock(PyListObject *, Py_ssize_t, _PyStackRef *); +#endif #define _PyList_ITEMS(op) _Py_RVALUE(_PyList_CAST(op)->ob_item) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index eda9b53629c..152238d21e5 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -2094,7 +2094,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -2331,7 +2331,7 @@ _PyOpcode_macro_expansion[256] = { [FORMAT_WITH_SPEC] = { .nuops = 1, .uops = { { _FORMAT_WITH_SPEC, OPARG_SIMPLE, 0 } } }, [FOR_ITER] = { .nuops = 1, .uops = { { _FOR_ITER, OPARG_REPLACED, 0 } } }, [FOR_ITER_GEN] = { .nuops = 3, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _FOR_ITER_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, - [FOR_ITER_LIST] = { .nuops = 3, .uops = { { _ITER_CHECK_LIST, OPARG_SIMPLE, 1 }, { _ITER_JUMP_LIST, OPARG_REPLACED, 1 }, { _ITER_NEXT_LIST, OPARG_SIMPLE, 1 } } }, + [FOR_ITER_LIST] = { .nuops = 3, .uops = { { _ITER_CHECK_LIST, OPARG_SIMPLE, 1 }, { _ITER_JUMP_LIST, OPARG_REPLACED, 1 }, { _ITER_NEXT_LIST, OPARG_REPLACED, 1 } } }, [FOR_ITER_RANGE] = { .nuops = 3, .uops = { { _ITER_CHECK_RANGE, OPARG_SIMPLE, 1 }, { _ITER_JUMP_RANGE, OPARG_REPLACED, 1 }, { _ITER_NEXT_RANGE, OPARG_SIMPLE, 1 } } }, [FOR_ITER_TUPLE] = { .nuops = 3, .uops = { { _ITER_CHECK_TUPLE, OPARG_SIMPLE, 1 }, { _ITER_JUMP_TUPLE, OPARG_REPLACED, 1 }, { _ITER_NEXT_TUPLE, OPARG_SIMPLE, 1 } } }, [GET_AITER] = { .nuops = 1, .uops = { { _GET_AITER, OPARG_SIMPLE, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 5143b10def5..095fd043090 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -164,126 +164,127 @@ extern "C" { #define _ITER_JUMP_RANGE 398 #define _ITER_JUMP_TUPLE 399 #define _ITER_NEXT_LIST 400 -#define _ITER_NEXT_RANGE 401 -#define _ITER_NEXT_TUPLE 402 -#define _JUMP_TO_TOP 403 +#define _ITER_NEXT_LIST_TIER_TWO 401 +#define _ITER_NEXT_RANGE 402 +#define _ITER_NEXT_TUPLE 403 +#define _JUMP_TO_TOP 404 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 404 -#define _LOAD_ATTR_CLASS 405 +#define _LOAD_ATTR 405 +#define _LOAD_ATTR_CLASS 406 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 406 -#define _LOAD_ATTR_METHOD_LAZY_DICT 407 -#define _LOAD_ATTR_METHOD_NO_DICT 408 -#define _LOAD_ATTR_METHOD_WITH_VALUES 409 -#define _LOAD_ATTR_MODULE 410 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 411 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 412 -#define _LOAD_ATTR_PROPERTY_FRAME 413 -#define _LOAD_ATTR_SLOT 414 -#define _LOAD_ATTR_WITH_HINT 415 +#define _LOAD_ATTR_INSTANCE_VALUE 407 +#define _LOAD_ATTR_METHOD_LAZY_DICT 408 +#define _LOAD_ATTR_METHOD_NO_DICT 409 +#define _LOAD_ATTR_METHOD_WITH_VALUES 410 +#define _LOAD_ATTR_MODULE 411 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 412 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 413 +#define _LOAD_ATTR_PROPERTY_FRAME 414 +#define _LOAD_ATTR_SLOT 415 +#define _LOAD_ATTR_WITH_HINT 416 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 416 +#define _LOAD_BYTECODE 417 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST #define _LOAD_CONST_IMMORTAL LOAD_CONST_IMMORTAL -#define _LOAD_CONST_INLINE 417 -#define _LOAD_CONST_INLINE_BORROW 418 +#define _LOAD_CONST_INLINE 418 +#define _LOAD_CONST_INLINE_BORROW 419 #define _LOAD_CONST_MORTAL LOAD_CONST_MORTAL #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 419 -#define _LOAD_FAST_0 420 -#define _LOAD_FAST_1 421 -#define _LOAD_FAST_2 422 -#define _LOAD_FAST_3 423 -#define _LOAD_FAST_4 424 -#define _LOAD_FAST_5 425 -#define _LOAD_FAST_6 426 -#define _LOAD_FAST_7 427 +#define _LOAD_FAST 420 +#define _LOAD_FAST_0 421 +#define _LOAD_FAST_1 422 +#define _LOAD_FAST_2 423 +#define _LOAD_FAST_3 424 +#define _LOAD_FAST_4 425 +#define _LOAD_FAST_5 426 +#define _LOAD_FAST_6 427 +#define _LOAD_FAST_7 428 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 428 -#define _LOAD_GLOBAL_BUILTINS 429 -#define _LOAD_GLOBAL_MODULE 430 +#define _LOAD_GLOBAL 429 +#define _LOAD_GLOBAL_BUILTINS 430 +#define _LOAD_GLOBAL_MODULE 431 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 431 -#define _LOAD_SMALL_INT_0 432 -#define _LOAD_SMALL_INT_1 433 -#define _LOAD_SMALL_INT_2 434 -#define _LOAD_SMALL_INT_3 435 +#define _LOAD_SMALL_INT 432 +#define _LOAD_SMALL_INT_0 433 +#define _LOAD_SMALL_INT_1 434 +#define _LOAD_SMALL_INT_2 435 +#define _LOAD_SMALL_INT_3 436 #define _LOAD_SPECIAL LOAD_SPECIAL #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 436 +#define _MAKE_CALLARGS_A_TUPLE 437 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 437 +#define _MAKE_WARM 438 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 438 -#define _MAYBE_EXPAND_METHOD_KW 439 -#define _MONITOR_CALL 440 -#define _MONITOR_CALL_KW 441 -#define _MONITOR_JUMP_BACKWARD 442 -#define _MONITOR_RESUME 443 +#define _MAYBE_EXPAND_METHOD 439 +#define _MAYBE_EXPAND_METHOD_KW 440 +#define _MONITOR_CALL 441 +#define _MONITOR_CALL_KW 442 +#define _MONITOR_JUMP_BACKWARD 443 +#define _MONITOR_RESUME 444 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 444 -#define _POP_JUMP_IF_TRUE 445 +#define _POP_JUMP_IF_FALSE 445 +#define _POP_JUMP_IF_TRUE 446 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE 446 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 447 +#define _POP_TOP_LOAD_CONST_INLINE 447 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 448 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 448 +#define _PUSH_FRAME 449 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 449 -#define _PY_FRAME_GENERAL 450 -#define _PY_FRAME_KW 451 -#define _QUICKEN_RESUME 452 -#define _REPLACE_WITH_TRUE 453 +#define _PUSH_NULL_CONDITIONAL 450 +#define _PY_FRAME_GENERAL 451 +#define _PY_FRAME_KW 452 +#define _QUICKEN_RESUME 453 +#define _REPLACE_WITH_TRUE 454 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 454 -#define _SEND 455 -#define _SEND_GEN_FRAME 456 +#define _SAVE_RETURN_OFFSET 455 +#define _SEND 456 +#define _SEND_GEN_FRAME 457 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 457 -#define _STORE_ATTR 458 -#define _STORE_ATTR_INSTANCE_VALUE 459 -#define _STORE_ATTR_SLOT 460 -#define _STORE_ATTR_WITH_HINT 461 +#define _START_EXECUTOR 458 +#define _STORE_ATTR 459 +#define _STORE_ATTR_INSTANCE_VALUE 460 +#define _STORE_ATTR_SLOT 461 +#define _STORE_ATTR_WITH_HINT 462 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 462 -#define _STORE_FAST_0 463 -#define _STORE_FAST_1 464 -#define _STORE_FAST_2 465 -#define _STORE_FAST_3 466 -#define _STORE_FAST_4 467 -#define _STORE_FAST_5 468 -#define _STORE_FAST_6 469 -#define _STORE_FAST_7 470 +#define _STORE_FAST 463 +#define _STORE_FAST_0 464 +#define _STORE_FAST_1 465 +#define _STORE_FAST_2 466 +#define _STORE_FAST_3 467 +#define _STORE_FAST_4 468 +#define _STORE_FAST_5 469 +#define _STORE_FAST_6 470 +#define _STORE_FAST_7 471 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 471 -#define _STORE_SUBSCR 472 +#define _STORE_SLICE 472 +#define _STORE_SUBSCR 473 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 473 -#define _TO_BOOL 474 +#define _TIER2_RESUME_CHECK 474 +#define _TO_BOOL 475 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -293,13 +294,13 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 475 +#define _UNPACK_SEQUENCE 476 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 475 +#define MAX_UOP_ID 476 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 883a07a408c..7f52352913a 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -185,7 +185,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_FOR_ITER_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_LIST] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_LIST] = HAS_EXIT_FLAG, - [_ITER_NEXT_LIST] = 0, + [_ITER_NEXT_LIST_TIER_TWO] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_TUPLE] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_TUPLE] = HAS_EXIT_FLAG, [_ITER_NEXT_TUPLE] = 0, @@ -428,7 +428,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", [_ITER_CHECK_RANGE] = "_ITER_CHECK_RANGE", [_ITER_CHECK_TUPLE] = "_ITER_CHECK_TUPLE", - [_ITER_NEXT_LIST] = "_ITER_NEXT_LIST", + [_ITER_NEXT_LIST_TIER_TWO] = "_ITER_NEXT_LIST_TIER_TWO", [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", [_ITER_NEXT_TUPLE] = "_ITER_NEXT_TUPLE", [_JUMP_TO_TOP] = "_JUMP_TO_TOP", @@ -889,7 +889,7 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GUARD_NOT_EXHAUSTED_LIST: return 0; - case _ITER_NEXT_LIST: + case _ITER_NEXT_LIST_TIER_TWO: return 0; case _ITER_CHECK_TUPLE: return 0; diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index e4224b843b2..946a4827fe7 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1802,5 +1802,44 @@ class TestSpecializer(TestBase): self.assert_specialized(load_const, "LOAD_CONST_MORTAL") self.assert_no_opcode(load_const, "LOAD_CONST") + @cpython_only + @requires_specialization_ft + def test_for_iter(self): + L = list(range(10)) + def for_iter_list(): + for i in L: + self.assertIn(i, L) + + for_iter_list() + self.assert_specialized(for_iter_list, "FOR_ITER_LIST") + self.assert_no_opcode(for_iter_list, "FOR_ITER") + + t = tuple(range(10)) + def for_iter_tuple(): + for i in t: + self.assertIn(i, t) + + for_iter_tuple() + self.assert_specialized(for_iter_tuple, "FOR_ITER_TUPLE") + self.assert_no_opcode(for_iter_tuple, "FOR_ITER") + + r = range(10) + def for_iter_range(): + for i in r: + self.assertIn(i, r) + + for_iter_range() + self.assert_specialized(for_iter_range, "FOR_ITER_RANGE") + self.assert_no_opcode(for_iter_range, "FOR_ITER") + + def for_iter_generator(): + for i in (i for i in range(10)): + i + 1 + + for_iter_generator() + self.assert_specialized(for_iter_generator, "FOR_ITER_GEN") + self.assert_no_opcode(for_iter_generator, "FOR_ITER") + + if __name__ == "__main__": unittest.main() diff --git a/Objects/listobject.c b/Objects/listobject.c index 4ad2b032f64..85eb5e56853 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -415,6 +415,32 @@ _PyList_GetItemRef(PyListObject *list, Py_ssize_t i) return list_get_item_ref(list, i); } +#ifdef Py_GIL_DISABLED +int +_PyList_GetItemRefNoLock(PyListObject *list, Py_ssize_t i, _PyStackRef *result) +{ + assert(_Py_IsOwnedByCurrentThread((PyObject *)list) || + _PyObject_GC_IS_SHARED(list)); + if (!valid_index(i, PyList_GET_SIZE(list))) { + return 0; + } + PyObject **ob_item = _Py_atomic_load_ptr(&list->ob_item); + if (ob_item == NULL) { + return 0; + } + Py_ssize_t cap = list_capacity(ob_item); + assert(cap != -1); + if (!valid_index(i, cap)) { + return 0; + } + PyObject *obj = _Py_atomic_load_ptr(&ob_item[i]); + if (obj == NULL || !_Py_TryIncrefCompareStackRef(&ob_item[i], obj, result)) { + return -1; + } + return 1; +} +#endif + int PyList_SetItem(PyObject *op, Py_ssize_t i, PyObject *newitem) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 17dc0c5bfb5..aaa25161d09 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3005,7 +3005,7 @@ dummy_func( }; specializing op(_SPECIALIZE_FOR_ITER, (counter/1, iter -- iter)) { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_ForIter(iter, next_instr, oparg); @@ -3013,7 +3013,7 @@ dummy_func( } OPCODE_DEFERRED_INC(FOR_ITER); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } replaced op(_FOR_ITER, (iter -- iter, next)) { @@ -3091,31 +3091,46 @@ dummy_func( op(_ITER_CHECK_LIST, (iter -- iter)) { - EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + EXIT_IF(Py_TYPE(iter_o) != &PyListIter_Type); +#ifdef Py_GIL_DISABLED + EXIT_IF(!_PyObject_IsUniquelyReferenced(iter_o)); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + EXIT_IF(!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) || + !_PyObject_GC_IS_SHARED(it->it_seq)); +#endif } replaced op(_ITER_JUMP_LIST, (iter -- iter)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyListIter_Type); +// For free-threaded Python, the loop exit can happen at any point during +// item retrieval, so it doesn't make much sense to check and jump +// separately before item retrieval. Any length check we do here can be +// invalid by the time we actually try to fetch the item. +#ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced(iter_o)); + (void)iter_o; +#else + _PyListIterObject *it = (_PyListIterObject *)iter_o; STAT_INC(FOR_ITER, hit); PyListObject *seq = it->it_seq; if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { it->it_index = -1; - #ifndef Py_GIL_DISABLED if (seq != NULL) { it->it_seq = NULL; Py_DECREF(seq); } - #endif /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); } +#endif } // Only used by Tier 2 op(_GUARD_NOT_EXHAUSTED_LIST, (iter -- iter)) { +#ifndef Py_GIL_DISABLED PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyListIterObject *it = (_PyListIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyListIter_Type); @@ -3125,16 +3140,62 @@ dummy_func( it->it_index = -1; EXIT_IF(1); } +#endif + } + + replaced op(_ITER_NEXT_LIST, (iter -- iter, next)) { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); + PyListObject *seq = it->it_seq; + assert(seq); +#ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced(iter_o)); + assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) || + _PyObject_GC_IS_SHARED(seq)); + STAT_INC(FOR_ITER, hit); + int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next); + // A negative result means we lost a race with another thread + // and we need to take the slow path. + DEOPT_IF(result < 0); + if (result == 0) { + it->it_index = -1; + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); + } + it->it_index++; +#else + assert(it->it_index < PyList_GET_SIZE(seq)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); +#endif } - op(_ITER_NEXT_LIST, (iter -- iter, next)) { + // Only used by Tier 2 + op(_ITER_NEXT_LIST_TIER_TWO, (iter -- iter, next)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyListIterObject *it = (_PyListIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyListIter_Type); PyListObject *seq = it->it_seq; assert(seq); +#ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced(iter_o)); + assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) || + _PyObject_GC_IS_SHARED(seq)); + STAT_INC(FOR_ITER, hit); + int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next); + // A negative result means we lost a race with another thread + // and we need to take the slow path. + EXIT_IF(result < 0); + if (result == 0) { + it->it_index = -1; + EXIT_IF(1); + } + it->it_index++; +#else assert(it->it_index < PyList_GET_SIZE(seq)); next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); +#endif } macro(FOR_ITER_LIST) = @@ -3144,20 +3205,30 @@ dummy_func( _ITER_NEXT_LIST; op(_ITER_CHECK_TUPLE, (iter -- iter)) { - EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + EXIT_IF(Py_TYPE(iter_o) != &PyTupleIter_Type); +#ifdef Py_GIL_DISABLED + EXIT_IF(!_PyObject_IsUniquelyReferenced(iter_o)); +#endif } replaced op(_ITER_JUMP_TUPLE, (iter -- iter)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + (void)iter_o; assert(Py_TYPE(iter_o) == &PyTupleIter_Type); +#ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced(iter_o)); +#endif + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; STAT_INC(FOR_ITER, hit); PyTupleObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { + if (seq == NULL || (size_t)it->it_index >= (size_t)PyTuple_GET_SIZE(seq)) { +#ifndef Py_GIL_DISABLED if (seq != NULL) { it->it_seq = NULL; Py_DECREF(seq); } +#endif /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); @@ -3169,6 +3240,9 @@ dummy_func( PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyTupleIter_Type); +#ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced(iter_o)); +#endif PyTupleObject *seq = it->it_seq; EXIT_IF(seq == NULL); EXIT_IF(it->it_index >= PyTuple_GET_SIZE(seq)); @@ -3179,6 +3253,9 @@ dummy_func( _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; +#ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced(iter_o)); +#endif assert(seq); assert(it->it_index < PyTuple_GET_SIZE(seq)); next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); @@ -3193,11 +3270,17 @@ dummy_func( op(_ITER_CHECK_RANGE, (iter -- iter)) { _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); EXIT_IF(Py_TYPE(r) != &PyRangeIter_Type); +#ifdef Py_GIL_DISABLED + EXIT_IF(!_PyObject_IsUniquelyReferenced((PyObject *)r)); +#endif } replaced op(_ITER_JUMP_RANGE, (iter -- iter)) { _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); +#ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); +#endif STAT_INC(FOR_ITER, hit); if (r->len <= 0) { // Jump over END_FOR instruction. @@ -3216,6 +3299,9 @@ dummy_func( op(_ITER_NEXT_RANGE, (iter -- iter, next)) { _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); +#ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); +#endif assert(r->len > 0); long value = r->start; r->start = value + r->step; @@ -3234,6 +3320,13 @@ dummy_func( op(_FOR_ITER_GEN_FRAME, (iter -- iter, gen_frame: _PyInterpreterFrame*)) { PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type); +#ifdef Py_GIL_DISABLED + // Since generators can't be used by multiple threads anyway we + // don't need to deopt here, but this lets us work on making + // generators thread-safe without necessarily having to + // specialize them thread-safely as well. + DEOPT_IF(!_PyObject_IsUniquelyReferenced((PyObject *)gen)); +#endif DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING); STAT_INC(FOR_ITER, hit); gen_frame = &gen->gi_iframe; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 8886564cded..fb5b2240436 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4171,10 +4171,23 @@ case _ITER_CHECK_LIST: { _PyStackRef iter; iter = stack_pointer[-1]; - if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type) { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyListIter_Type) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced(iter_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + _PyListIterObject *it = (_PyListIterObject *)iter_o; + if (!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) || + !_PyObject_GC_IS_SHARED(it->it_seq)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #endif break; } @@ -4183,6 +4196,7 @@ case _GUARD_NOT_EXHAUSTED_LIST: { _PyStackRef iter; iter = stack_pointer[-1]; + #ifndef Py_GIL_DISABLED PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyListIterObject *it = (_PyListIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyListIter_Type); @@ -4198,10 +4212,13 @@ JUMP_TO_JUMP_TARGET(); } } + #endif break; } - case _ITER_NEXT_LIST: { + /* _ITER_NEXT_LIST is not a viable micro-op for tier 2 because it is replaced */ + + case _ITER_NEXT_LIST_TIER_TWO: { _PyStackRef iter; _PyStackRef next; iter = stack_pointer[-1]; @@ -4210,8 +4227,32 @@ assert(Py_TYPE(iter_o) == &PyListIter_Type); PyListObject *seq = it->it_seq; assert(seq); + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced(iter_o)); + assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) || + _PyObject_GC_IS_SHARED(seq)); + STAT_INC(FOR_ITER, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next); + stack_pointer = _PyFrame_GetStackPointer(frame); + // A negative result means we lost a race with another thread + // and we need to take the slow path. + if (result < 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (result == 0) { + it->it_index = -1; + if (1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + it->it_index++; + #else assert(it->it_index < PyList_GET_SIZE(seq)); next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + #endif stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -4221,10 +4262,17 @@ case _ITER_CHECK_TUPLE: { _PyStackRef iter; iter = stack_pointer[-1]; - if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type) { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyTupleIter_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced(iter_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + #endif break; } @@ -4236,6 +4284,9 @@ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyTupleIter_Type); + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced(iter_o)); + #endif PyTupleObject *seq = it->it_seq; if (seq == NULL) { UOP_STAT_INC(uopcode, miss); @@ -4256,6 +4307,9 @@ _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced(iter_o)); + #endif assert(seq); assert(it->it_index < PyTuple_GET_SIZE(seq)); next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); @@ -4273,6 +4327,12 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #endif break; } @@ -4296,6 +4356,9 @@ iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); + #endif assert(r->len > 0); long value = r->start; r->start = value + r->step; @@ -4321,6 +4384,16 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + #ifdef Py_GIL_DISABLED + // Since generators can't be used by multiple threads anyway we + // don't need to deopt here, but this lets us work on making + // generators thread-safe without necessarily having to + // specialize them thread-safely as well. + if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #endif if (gen->gi_frame_state >= FRAME_EXECUTING) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ad0e4118817..8cdbaf1a4d5 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5434,7 +5434,7 @@ iter = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5444,7 +5444,7 @@ } OPCODE_DEFERRED_INC(FOR_ITER); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } // _FOR_ITER { @@ -5514,6 +5514,17 @@ assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } + #ifdef Py_GIL_DISABLED + // Since generators can't be used by multiple threads anyway we + // don't need to deopt here, but this lets us work on making + // generators thread-safe without necessarily having to + // specialize them thread-safely as well. + if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + #endif if (gen->gi_frame_state >= FRAME_EXECUTING) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); @@ -5565,33 +5576,55 @@ // _ITER_CHECK_LIST { iter = stack_pointer[-1]; - if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type) { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyListIter_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced(iter_o)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + _PyListIterObject *it = (_PyListIterObject *)iter_o; + if (!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) || + !_PyObject_GC_IS_SHARED(it->it_seq)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + #endif } // _ITER_JUMP_LIST { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyListIter_Type); + // For free-threaded Python, the loop exit can happen at any point during + // item retrieval, so it doesn't make much sense to check and jump + // separately before item retrieval. Any length check we do here can be + // invalid by the time we actually try to fetch the item. + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced(iter_o)); + (void)iter_o; + #else + _PyListIterObject *it = (_PyListIterObject *)iter_o; STAT_INC(FOR_ITER, hit); PyListObject *seq = it->it_seq; if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { it->it_index = -1; - #ifndef Py_GIL_DISABLED if (seq != NULL) { it->it_seq = NULL; _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(seq); stack_pointer = _PyFrame_GetStackPointer(frame); } - #endif /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); } + #endif } // _ITER_NEXT_LIST { @@ -5600,8 +5633,32 @@ assert(Py_TYPE(iter_o) == &PyListIter_Type); PyListObject *seq = it->it_seq; assert(seq); + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced(iter_o)); + assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) || + _PyObject_GC_IS_SHARED(seq)); + STAT_INC(FOR_ITER, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next); + stack_pointer = _PyFrame_GetStackPointer(frame); + // A negative result means we lost a race with another thread + // and we need to take the slow path. + if (result < 0) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + if (result == 0) { + it->it_index = -1; + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); + } + it->it_index++; + #else assert(it->it_index < PyList_GET_SIZE(seq)); next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + #endif } stack_pointer[0] = next; stack_pointer += 1; @@ -5632,11 +5689,21 @@ assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + #endif } // _ITER_JUMP_RANGE { _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); + #endif STAT_INC(FOR_ITER, hit); if (r->len <= 0) { // Jump over END_FOR instruction. @@ -5648,6 +5715,9 @@ { _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); + #endif assert(r->len > 0); long value = r->start; r->start = value + r->step; @@ -5681,26 +5751,40 @@ // _ITER_CHECK_TUPLE { iter = stack_pointer[-1]; - if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type) { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyTupleIter_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced(iter_o)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + #endif } // _ITER_JUMP_TUPLE { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + (void)iter_o; assert(Py_TYPE(iter_o) == &PyTupleIter_Type); + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced(iter_o)); + #endif + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; STAT_INC(FOR_ITER, hit); PyTupleObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { + if (seq == NULL || (size_t)it->it_index >= (size_t)PyTuple_GET_SIZE(seq)) { + #ifndef Py_GIL_DISABLED if (seq != NULL) { it->it_seq = NULL; _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(seq); stack_pointer = _PyFrame_GetStackPointer(frame); } + #endif /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); @@ -5712,6 +5796,9 @@ _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced(iter_o)); + #endif assert(seq); assert(it->it_index < PyTuple_GET_SIZE(seq)); next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); diff --git a/Python/optimizer.c b/Python/optimizer.c index e05523451da..6fc5eabdf8b 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -363,6 +363,7 @@ _PyUOp_Replacements[MAX_UOP_ID + 1] = { [_ITER_JUMP_LIST] = _GUARD_NOT_EXHAUSTED_LIST, [_ITER_JUMP_TUPLE] = _GUARD_NOT_EXHAUSTED_TUPLE, [_FOR_ITER] = _FOR_ITER_TIER_TWO, + [_ITER_NEXT_LIST] = _ITER_NEXT_LIST_TIER_TWO, }; static const uint8_t diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 9f3fc052494..3f315901a5b 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1458,7 +1458,9 @@ break; } - case _ITER_NEXT_LIST: { + /* _ITER_NEXT_LIST is not a viable micro-op for tier 2 */ + + case _ITER_NEXT_LIST_TIER_TWO: { JitOptSymbol *next; next = sym_new_not_null(ctx); stack_pointer[0] = next; diff --git a/Python/specialize.c b/Python/specialize.c index c741c4f93f3..0466b5bee90 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2826,45 +2826,56 @@ int void _Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg) { - assert(ENABLE_SPECIALIZATION); + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[FOR_ITER] == INLINE_CACHE_ENTRIES_FOR_ITER); - _PyForIterCache *cache = (_PyForIterCache *)(instr + 1); PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); PyTypeObject *tp = Py_TYPE(iter_o); +#ifdef Py_GIL_DISABLED + // Only specialize for uniquely referenced iterators, so that we know + // they're only referenced by this one thread. This is more limiting + // than we need (even `it = iter(mylist); for item in it:` won't get + // specialized) but we don't have a way to check whether we're the only + // _thread_ who has access to the object. + if (!_PyObject_IsUniquelyReferenced(iter_o)) + goto failure; +#endif if (tp == &PyListIter_Type) { - instr->op.code = FOR_ITER_LIST; - goto success; +#ifdef Py_GIL_DISABLED + _PyListIterObject *it = (_PyListIterObject *)iter_o; + if (!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) && + !_PyObject_GC_IS_SHARED(it->it_seq)) { + // Maybe this should just set GC_IS_SHARED in a critical + // section, instead of leaving it to the first iteration? + goto failure; + } +#endif + specialize(instr, FOR_ITER_LIST); + return; } else if (tp == &PyTupleIter_Type) { - instr->op.code = FOR_ITER_TUPLE; - goto success; + specialize(instr, FOR_ITER_TUPLE); + return; } else if (tp == &PyRangeIter_Type) { - instr->op.code = FOR_ITER_RANGE; - goto success; + specialize(instr, FOR_ITER_RANGE); + return; } else if (tp == &PyGen_Type && oparg <= SHRT_MAX) { + // Generators are very much not thread-safe, so don't worry about + // the specialization not being thread-safe. assert(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == END_FOR || instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == INSTRUMENTED_END_FOR ); /* Don't specialize if PEP 523 is active */ - if (_PyInterpreterState_GET()->eval_frame) { - SPECIALIZATION_FAIL(FOR_ITER, SPEC_FAIL_OTHER); + if (_PyInterpreterState_GET()->eval_frame) goto failure; - } - instr->op.code = FOR_ITER_GEN; - goto success; + specialize(instr, FOR_ITER_GEN); + return; } +failure: SPECIALIZATION_FAIL(FOR_ITER, _PySpecialization_ClassifyIterator(iter_o)); -failure: - STAT_INC(FOR_ITER, failure); - instr->op.code = FOR_ITER; - cache->counter = adaptive_counter_backoff(cache->counter); - return; -success: - STAT_INC(FOR_ITER, success); - cache->counter = adaptive_counter_cooldown(); + unspecialize(instr); } void diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index faf720e8fe6..664e247226c 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -644,11 +644,13 @@ NON_ESCAPING_FUNCTIONS = ( "_PyLong_IsNonNegativeCompact", "_PyLong_IsZero", "_PyManagedDictPointer_IsValues", + "_PyObject_GC_IS_SHARED", "_PyObject_GC_IS_TRACKED", "_PyObject_GC_MAY_BE_TRACKED", "_PyObject_GC_TRACK", "_PyObject_GetManagedDict", "_PyObject_InlineValues", + "_PyObject_IsUniquelyReferenced", "_PyObject_ManagedDictPointer", "_PyThreadState_HasStackSpace", "_PyTuple_FromStackRefStealOnSuccess", @@ -661,6 +663,7 @@ NON_ESCAPING_FUNCTIONS = ( "_Py_DECREF_NO_DEALLOC", "_Py_ID", "_Py_IsImmortal", + "_Py_IsOwnedByCurrentThread", "_Py_LeaveRecursiveCallPy", "_Py_LeaveRecursiveCallTstate", "_Py_NewRef", |