bpo-46841: Use *inline* caching for `BINARY_OP` (GH-31543)

5 files changed, 143 insertions, 124 deletions

diff --git a/Python/ceval.c b/Python/ceval.c
index a64a24f8a0a..c09914fadca 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -1939,6 +1939,7 @@ handle_eval_breaker:
             if (prod == NULL) {
                 goto error;
             }
+            JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
             NOTRACE_DISPATCH();
         }
 
@@ -1959,6 +1960,7 @@ handle_eval_breaker:
             if (prod == NULL) {
                 goto error;
             }
+            JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
             NOTRACE_DISPATCH();
         }
 
@@ -1977,6 +1979,7 @@ handle_eval_breaker:
             if (sub == NULL) {
                 goto error;
             }
+            JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
             NOTRACE_DISPATCH();
         }
 
@@ -1996,6 +1999,7 @@ handle_eval_breaker:
             if (sub == NULL) {
                 goto error;
             }
+            JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
             NOTRACE_DISPATCH();
         }
 
@@ -2014,6 +2018,7 @@ handle_eval_breaker:
             if (TOP() == NULL) {
                 goto error;
             }
+            JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
             NOTRACE_DISPATCH();
         }
 
@@ -2043,6 +2048,7 @@ handle_eval_breaker:
             if (TOP() == NULL) {
                 goto error;
             }
+            JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
             NOTRACE_DISPATCH();
         }
 
@@ -2063,6 +2069,7 @@ handle_eval_breaker:
             if (sum == NULL) {
                 goto error;
             }
+            JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
             NOTRACE_DISPATCH();
         }
 
@@ -2081,6 +2088,7 @@ handle_eval_breaker:
             if (sum == NULL) {
                 goto error;
             }
+            JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
             NOTRACE_DISPATCH();
         }
 
@@ -5425,23 +5433,23 @@ handle_eval_breaker:
             if (res == NULL) {
                 goto error;
             }
+            JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
             DISPATCH();
         }
 
         TARGET(BINARY_OP_ADAPTIVE) {
             assert(cframe.use_tracing == 0);
-            SpecializedCacheEntry *cache = GET_CACHE();
-            if (cache->adaptive.counter == 0) {
+            _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr;
+            if (cache->counter == 0) {
                 PyObject *lhs = SECOND();
                 PyObject *rhs = TOP();
                 next_instr--;
-                _Py_Specialize_BinaryOp(lhs, rhs, next_instr, cache);
+                _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg);
                 DISPATCH();
             }
             else {
                 STAT_INC(BINARY_OP, deferred);
-                cache->adaptive.counter--;
-                oparg = cache->adaptive.original_oparg;
+                cache->counter--;
                 JUMP_TO_INSTRUCTION(BINARY_OP);
             }
         }
@@ -5462,6 +5470,10 @@ handle_eval_breaker:
             DISPATCH_GOTO();
         }
 
+        TARGET(CACHE) {
+            Py_UNREACHABLE();
+        }
+
 #if USE_COMPUTED_GOTOS
         TARGET_DO_TRACING: {
 #else
@@ -5548,6 +5560,22 @@ opname ## _miss: \
         JUMP_TO_INSTRUCTION(opname); \
     }
 
+#define MISS_WITH_INLINE_CACHE(opname) \
+opname ## _miss: \
+    { \
+        STAT_INC(opcode, miss); \
+        STAT_INC(opname, miss); \
+        /* The counter is always the first cache entry: */ \
+        _Py_CODEUNIT *counter = (_Py_CODEUNIT *)next_instr; \
+        *counter -= 1; \
+        if (*counter == 0) { \
+            next_instr[-1] = _Py_MAKECODEUNIT(opname ## _ADAPTIVE, _Py_OPARG(next_instr[-1])); \
+            STAT_INC(opname, deopt); \
+            *counter = ADAPTIVE_CACHE_BACKOFF; \
+        } \
+        JUMP_TO_INSTRUCTION(opname); \
+    }
+
 #define MISS_WITH_OPARG_COUNTER(opname) \
 opname ## _miss: \
     { \
@@ -5569,7 +5597,7 @@ MISS_WITH_CACHE(LOAD_GLOBAL)
 MISS_WITH_CACHE(LOAD_METHOD)
 MISS_WITH_CACHE(PRECALL)
 MISS_WITH_CACHE(CALL)
-MISS_WITH_CACHE(BINARY_OP)
+MISS_WITH_INLINE_CACHE(BINARY_OP)
 MISS_WITH_CACHE(COMPARE_OP)
 MISS_WITH_CACHE(BINARY_SUBSCR)
 MISS_WITH_CACHE(UNPACK_SEQUENCE)
diff --git a/Python/compile.c b/Python/compile.c
index cb6e5e3550f..40e6263ff7d 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -33,7 +33,6 @@
 
 #define NEED_OPCODE_JUMP_TABLES
 #include "opcode.h"               // EXTENDED_ARG
-#include "wordcode_helpers.h"     // instrsize()
 
 
 #define DEFAULT_BLOCK_SIZE 16
@@ -131,6 +130,43 @@ is_jump(struct instr *i)
     return i->i_opcode >= SETUP_WITH || is_bit_set_in_table(_PyOpcode_Jump, i->i_opcode);
 }
 
+static int
+instr_size(struct instr *instruction)
+{
+    int opcode = instruction->i_opcode;
+    int oparg = instruction->i_oparg;
+    int extended_args = (0xFFFFFF < oparg) + (0xFFFF < oparg) + (0xFF < oparg);
+    int caches = _PyOpcode_InlineCacheEntries[opcode];
+    return extended_args + 1 + caches;
+}
+
+static void
+write_instr(_Py_CODEUNIT *codestr, struct instr *instruction, int ilen)
+{
+    int opcode = instruction->i_opcode;
+    int oparg = instruction->i_oparg;
+    int caches = _PyOpcode_InlineCacheEntries[opcode];
+    switch (ilen - caches) {
+        case 4:
+            *codestr++ = _Py_MAKECODEUNIT(EXTENDED_ARG, (oparg >> 24) & 0xFF);
+            /* fall through */
+        case 3:
+            *codestr++ = _Py_MAKECODEUNIT(EXTENDED_ARG, (oparg >> 16) & 0xFF);
+            /* fall through */
+        case 2:
+            *codestr++ = _Py_MAKECODEUNIT(EXTENDED_ARG, (oparg >> 8) & 0xFF);
+            /* fall through */
+        case 1:
+            *codestr++ = _Py_MAKECODEUNIT(opcode, oparg & 0xFF);
+            break;
+        default:
+            Py_UNREACHABLE();
+    }
+    while (caches--) {
+        *codestr++ = _Py_MAKECODEUNIT(CACHE, 0);
+    }
+}
+
 typedef struct basicblock_ {
     /* Each basicblock in a compilation unit is linked via b_list in the
        reverse order that the block are allocated.  b_list points to the next
@@ -854,6 +890,7 @@ stack_effect(int opcode, int oparg, int jump)
         case NOP:
         case EXTENDED_ARG:
         case RESUME:
+        case CACHE:
             return 0;
 
         /* Stack manipulation */
@@ -7065,8 +7102,9 @@ blocksize(basicblock *b)
     int i;
     int size = 0;
 
-    for (i = 0; i < b->b_iused; i++)
-        size += instrsize(b->b_instr[i].i_oparg);
+    for (i = 0; i < b->b_iused; i++) {
+        size += instr_size(&b->b_instr[i]);
+    }
     return size;
 }
 
@@ -7330,7 +7368,7 @@ assemble_exception_table(struct assembler *a)
                 start = ioffset;
                 handler = instr->i_except;
             }
-            ioffset += instrsize(instr->i_oparg);
+            ioffset += instr_size(instr);
         }
     }
     if (handler != NULL) {
@@ -7459,12 +7497,10 @@ assemble_cnotab(struct assembler* a, struct instr* i, int instr_size)
 static int
 assemble_emit(struct assembler *a, struct instr *i)
 {
-    int size, arg = 0;
     Py_ssize_t len = PyBytes_GET_SIZE(a->a_bytecode);
     _Py_CODEUNIT *code;
 
-    arg = i->i_oparg;
-    size = instrsize(arg);
+    int size = instr_size(i);
     if (i->i_lineno && !assemble_lnotab(a, i)) {
         return 0;
     }
@@ -7482,7 +7518,7 @@ assemble_emit(struct assembler *a, struct instr *i)
     }
     code = (_Py_CODEUNIT *)PyBytes_AS_STRING(a->a_bytecode) + a->a_offset;
     a->a_offset += size;
-    write_op_arg(code, i->i_opcode, arg, size);
+    write_instr(code, i, size);
     return 1;
 }
 
@@ -7532,7 +7568,7 @@ assemble_jump_offsets(struct assembler *a, struct compiler *c)
             bsize = b->b_offset;
             for (i = 0; i < b->b_iused; i++) {
                 struct instr *instr = &b->b_instr[i];
-                int isize = instrsize(instr->i_oparg);
+                int isize = instr_size(instr);
                 /* Relative jumps are computed relative to
                    the instruction pointer after fetching
                    the jump instruction.
@@ -7543,7 +7579,7 @@ assemble_jump_offsets(struct assembler *a, struct compiler *c)
                     if (is_relative_jump(instr)) {
                         instr->i_oparg -= bsize;
                     }
-                    if (instrsize(instr->i_oparg) != isize) {
+                    if (instr_size(instr) != isize) {
                         extended_arg_recompile = 1;
                     }
                 }
@@ -7555,7 +7591,7 @@ assemble_jump_offsets(struct assembler *a, struct compiler *c)
         with a better solution.
 
         The issue is that in the first loop blocksize() is called
-        which calls instrsize() which requires i_oparg be set
+        which calls instr_size() which requires i_oparg be set
         appropriately. There is a bootstrap problem because
         i_oparg is calculated in the second loop above.
 
diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h
index 3339ab1a043..d463e303e27 100644
--- a/Python/opcode_targets.h
+++ b/Python/opcode_targets.h
@@ -2,19 +2,20 @@ static void *opcode_targets[256] = {
     &&_unknown_opcode,
     &&TARGET_POP_TOP,
     &&TARGET_PUSH_NULL,
+    &&TARGET_CACHE,
     &&TARGET_BINARY_OP_ADAPTIVE,
     &&TARGET_BINARY_OP_ADD_INT,
     &&TARGET_BINARY_OP_ADD_FLOAT,
     &&TARGET_BINARY_OP_ADD_UNICODE,
     &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE,
-    &&TARGET_BINARY_OP_MULTIPLY_INT,
     &&TARGET_NOP,
     &&TARGET_UNARY_POSITIVE,
     &&TARGET_UNARY_NEGATIVE,
     &&TARGET_UNARY_NOT,
+    &&TARGET_BINARY_OP_MULTIPLY_INT,
     &&TARGET_BINARY_OP_MULTIPLY_FLOAT,
-    &&TARGET_BINARY_OP_SUBTRACT_INT,
     &&TARGET_UNARY_INVERT,
+    &&TARGET_BINARY_OP_SUBTRACT_INT,
     &&TARGET_BINARY_OP_SUBTRACT_FLOAT,
     &&TARGET_COMPARE_OP_ADAPTIVE,
     &&TARGET_COMPARE_OP_FLOAT_JUMP,
@@ -23,18 +24,18 @@ static void *opcode_targets[256] = {
     &&TARGET_BINARY_SUBSCR_ADAPTIVE,
     &&TARGET_BINARY_SUBSCR_GETITEM,
     &&TARGET_BINARY_SUBSCR_LIST_INT,
-    &&TARGET_BINARY_SUBSCR_TUPLE_INT,
     &&TARGET_BINARY_SUBSCR,
+    &&TARGET_BINARY_SUBSCR_TUPLE_INT,
     &&TARGET_BINARY_SUBSCR_DICT,
     &&TARGET_STORE_SUBSCR_ADAPTIVE,
     &&TARGET_STORE_SUBSCR_LIST_INT,
-    &&TARGET_STORE_SUBSCR_DICT,
     &&TARGET_GET_LEN,
     &&TARGET_MATCH_MAPPING,
     &&TARGET_MATCH_SEQUENCE,
     &&TARGET_MATCH_KEYS,
-    &&TARGET_CALL_ADAPTIVE,
+    &&TARGET_STORE_SUBSCR_DICT,
     &&TARGET_PUSH_EXC_INFO,
+    &&TARGET_CALL_ADAPTIVE,
     &&TARGET_CALL_PY_EXACT_ARGS,
     &&TARGET_CALL_PY_WITH_DEFAULTS,
     &&TARGET_JUMP_ABSOLUTE_QUICK,
@@ -47,40 +48,39 @@ static void *opcode_targets[256] = {
     &&TARGET_LOAD_GLOBAL_MODULE,
     &&TARGET_LOAD_GLOBAL_BUILTIN,
     &&TARGET_LOAD_METHOD_ADAPTIVE,
-    &&TARGET_LOAD_METHOD_CLASS,
     &&TARGET_WITH_EXCEPT_START,
     &&TARGET_GET_AITER,
     &&TARGET_GET_ANEXT,
     &&TARGET_BEFORE_ASYNC_WITH,
     &&TARGET_BEFORE_WITH,
     &&TARGET_END_ASYNC_FOR,
+    &&TARGET_LOAD_METHOD_CLASS,
     &&TARGET_LOAD_METHOD_MODULE,
     &&TARGET_LOAD_METHOD_NO_DICT,
     &&TARGET_LOAD_METHOD_WITH_DICT,
     &&TARGET_LOAD_METHOD_WITH_VALUES,
-    &&TARGET_PRECALL_ADAPTIVE,
     &&TARGET_STORE_SUBSCR,
     &&TARGET_DELETE_SUBSCR,
+    &&TARGET_PRECALL_ADAPTIVE,
     &&TARGET_PRECALL_BUILTIN_CLASS,
     &&TARGET_PRECALL_NO_KW_BUILTIN_O,
     &&TARGET_PRECALL_NO_KW_BUILTIN_FAST,
     &&TARGET_PRECALL_BUILTIN_FAST_WITH_KEYWORDS,
     &&TARGET_PRECALL_NO_KW_LEN,
-    &&TARGET_PRECALL_NO_KW_ISINSTANCE,
     &&TARGET_GET_ITER,
     &&TARGET_GET_YIELD_FROM_ITER,
     &&TARGET_PRINT_EXPR,
     &&TARGET_LOAD_BUILD_CLASS,
-    &&TARGET_PRECALL_NO_KW_LIST_APPEND,
+    &&TARGET_PRECALL_NO_KW_ISINSTANCE,
     &&TARGET_GET_AWAITABLE,
     &&TARGET_LOAD_ASSERTION_ERROR,
     &&TARGET_RETURN_GENERATOR,
+    &&TARGET_PRECALL_NO_KW_LIST_APPEND,
     &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_O,
     &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS,
     &&TARGET_PRECALL_NO_KW_STR_1,
     &&TARGET_PRECALL_NO_KW_TUPLE_1,
     &&TARGET_PRECALL_NO_KW_TYPE_1,
-    &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST,
     &&TARGET_LIST_TO_TUPLE,
     &&TARGET_RETURN_VALUE,
     &&TARGET_IMPORT_STAR,
@@ -130,7 +130,7 @@ static void *opcode_targets[256] = {
     &&TARGET_POP_JUMP_IF_NOT_NONE,
     &&TARGET_POP_JUMP_IF_NONE,
     &&TARGET_RAISE_VARARGS,
-    &&TARGET_PRECALL_BOUND_METHOD,
+    &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST,
     &&TARGET_MAKE_FUNCTION,
     &&TARGET_BUILD_SLICE,
     &&TARGET_JUMP_NO_INTERRUPT,
@@ -139,39 +139,40 @@ static void *opcode_targets[256] = {
     &&TARGET_LOAD_DEREF,
     &&TARGET_STORE_DEREF,
     &&TARGET_DELETE_DEREF,
+    &&TARGET_PRECALL_BOUND_METHOD,
     &&TARGET_PRECALL_PYFUNC,
-    &&TARGET_RESUME_QUICK,
     &&TARGET_CALL_FUNCTION_EX,
-    &&TARGET_STORE_ATTR_ADAPTIVE,
+    &&TARGET_RESUME_QUICK,
     &&TARGET_EXTENDED_ARG,
     &&TARGET_LIST_APPEND,
     &&TARGET_SET_ADD,
     &&TARGET_MAP_ADD,
     &&TARGET_LOAD_CLASSDEREF,
     &&TARGET_COPY_FREE_VARS,
-    &&TARGET_STORE_ATTR_INSTANCE_VALUE,
+    &&TARGET_STORE_ATTR_ADAPTIVE,
     &&TARGET_RESUME,
     &&TARGET_MATCH_CLASS,
+    &&TARGET_STORE_ATTR_INSTANCE_VALUE,
     &&TARGET_STORE_ATTR_SLOT,
-    &&TARGET_STORE_ATTR_WITH_HINT,
     &&TARGET_FORMAT_VALUE,
     &&TARGET_BUILD_CONST_KEY_MAP,
     &&TARGET_BUILD_STRING,
+    &&TARGET_STORE_ATTR_WITH_HINT,
     &&TARGET_UNPACK_SEQUENCE_ADAPTIVE,
-    &&TARGET_UNPACK_SEQUENCE_LIST,
     &&TARGET_LOAD_METHOD,
-    &&TARGET_UNPACK_SEQUENCE_TUPLE,
+    &&TARGET_UNPACK_SEQUENCE_LIST,
     &&TARGET_LIST_EXTEND,
     &&TARGET_SET_UPDATE,
     &&TARGET_DICT_MERGE,
     &&TARGET_DICT_UPDATE,
     &&TARGET_PRECALL,
+    &&TARGET_UNPACK_SEQUENCE_TUPLE,
     &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE,
     &&TARGET_LOAD_FAST__LOAD_FAST,
     &&TARGET_STORE_FAST__LOAD_FAST,
-    &&TARGET_LOAD_FAST__LOAD_CONST,
     &&TARGET_CALL,
     &&TARGET_KW_NAMES,
+    &&TARGET_LOAD_FAST__LOAD_CONST,
     &&TARGET_LOAD_CONST__LOAD_FAST,
     &&TARGET_STORE_FAST__STORE_FAST,
     &&TARGET_LOAD_FAST__LOAD_ATTR_INSTANCE_VALUE,
@@ -253,6 +254,5 @@ static void *opcode_targets[256] = {
     &&_unknown_opcode,
     &&_unknown_opcode,
     &&_unknown_opcode,
-    &&_unknown_opcode,
     &&TARGET_DO_TRACING
 };
diff --git a/Python/specialize.c b/Python/specialize.c
index 6e4eb46443d..e1db12b853c 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -65,7 +65,6 @@ static uint8_t cache_requirements[256] = {
     [CALL] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */
     [PRECALL] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */
     [STORE_ATTR] = 1,  // _PyAdaptiveEntry
-    [BINARY_OP] = 1,  // _PyAdaptiveEntry
     [COMPARE_OP] = 1, /* _PyAdaptiveEntry */
     [UNPACK_SEQUENCE] = 1,  // _PyAdaptiveEntry
 };
@@ -385,29 +384,34 @@ optimize(SpecializedCacheOrInstruction *quickened, int len)
         int opcode = _Py_OPCODE(instructions[i]);
         int oparg = _Py_OPARG(instructions[i]);
         uint8_t adaptive_opcode = adaptive_opcodes[opcode];
-        if (adaptive_opcode && previous_opcode != EXTENDED_ARG) {
-            int new_oparg = oparg_from_instruction_and_update_offset(
-                i, opcode, oparg, &cache_offset
-            );
-            if (new_oparg < 0) {
-                /* Not possible to allocate a cache for this instruction */
-                previous_opcode = opcode;
-                continue;
+        if (adaptive_opcode) {
+            if (_PyOpcode_InlineCacheEntries[opcode]) {
+                instructions[i] = _Py_MAKECODEUNIT(adaptive_opcode, oparg);
             }
-            previous_opcode = adaptive_opcode;
-            int entries_needed = cache_requirements[opcode];
-            if (entries_needed) {
-                /* Initialize the adpative cache entry */
-                int cache0_offset = cache_offset-entries_needed;
-                SpecializedCacheEntry *cache =
-                    _GetSpecializedCacheEntry(instructions, cache0_offset);
-                cache->adaptive.original_oparg = oparg;
-                cache->adaptive.counter = 0;
-            } else {
-                // oparg is the adaptive cache counter
-                new_oparg = 0;
+            else if (previous_opcode != EXTENDED_ARG) {
+                int new_oparg = oparg_from_instruction_and_update_offset(
+                    i, opcode, oparg, &cache_offset
+                );
+                if (new_oparg < 0) {
+                    /* Not possible to allocate a cache for this instruction */
+                    previous_opcode = opcode;
+                    continue;
+                }
+                previous_opcode = adaptive_opcode;
+                int entries_needed = cache_requirements[opcode];
+                if (entries_needed) {
+                    /* Initialize the adpative cache entry */
+                    int cache0_offset = cache_offset-entries_needed;
+                    SpecializedCacheEntry *cache =
+                        _GetSpecializedCacheEntry(instructions, cache0_offset);
+                    cache->adaptive.original_oparg = oparg;
+                    cache->adaptive.counter = 0;
+                } else {
+                    // oparg is the adaptive cache counter
+                    new_oparg = 0;
+                }
+                instructions[i] = _Py_MAKECODEUNIT(adaptive_opcode, new_oparg);
             }
-            instructions[i] = _Py_MAKECODEUNIT(adaptive_opcode, new_oparg);
         }
         else {
             /* Super instructions don't use the cache,
@@ -1922,10 +1926,12 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs)
 
 void
 _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
-                        SpecializedCacheEntry *cache)
+                        int oparg)
 {
-    _PyAdaptiveEntry *adaptive = &cache->adaptive;
-    switch (adaptive->original_oparg) {
+    assert(_PyOpcode_InlineCacheEntries[BINARY_OP] == 
+           INLINE_CACHE_ENTRIES_BINARY_OP);
+    _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(instr + 1);
+    switch (oparg) {
         case NB_ADD:
         case NB_INPLACE_ADD:
             if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
@@ -1934,20 +1940,18 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
             if (PyUnicode_CheckExact(lhs)) {
                 if (_Py_OPCODE(instr[1]) == STORE_FAST && Py_REFCNT(lhs) == 2) {
                     *instr = _Py_MAKECODEUNIT(BINARY_OP_INPLACE_ADD_UNICODE,
-                                              _Py_OPARG(*instr));
+                                              oparg);
                     goto success;
                 }
-                *instr = _Py_MAKECODEUNIT(BINARY_OP_ADD_UNICODE,
-                                          _Py_OPARG(*instr));
+                *instr = _Py_MAKECODEUNIT(BINARY_OP_ADD_UNICODE, oparg);
                 goto success;
             }
             if (PyLong_CheckExact(lhs)) {
-                *instr = _Py_MAKECODEUNIT(BINARY_OP_ADD_INT, _Py_OPARG(*instr));
+                *instr = _Py_MAKECODEUNIT(BINARY_OP_ADD_INT, oparg);
                 goto success;
             }
             if (PyFloat_CheckExact(lhs)) {
-                *instr = _Py_MAKECODEUNIT(BINARY_OP_ADD_FLOAT,
-                                          _Py_OPARG(*instr));
+                *instr = _Py_MAKECODEUNIT(BINARY_OP_ADD_FLOAT, oparg);
                 goto success;
             }
             break;
@@ -1957,13 +1961,11 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
                 break;
             }
             if (PyLong_CheckExact(lhs)) {
-                *instr = _Py_MAKECODEUNIT(BINARY_OP_MULTIPLY_INT,
-                                          _Py_OPARG(*instr));
+                *instr = _Py_MAKECODEUNIT(BINARY_OP_MULTIPLY_INT, oparg);
                 goto success;
             }
             if (PyFloat_CheckExact(lhs)) {
-                *instr = _Py_MAKECODEUNIT(BINARY_OP_MULTIPLY_FLOAT,
-                                          _Py_OPARG(*instr));
+                *instr = _Py_MAKECODEUNIT(BINARY_OP_MULTIPLY_FLOAT, oparg);
                 goto success;
             }
             break;
@@ -1973,13 +1975,11 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
                 break;
             }
             if (PyLong_CheckExact(lhs)) {
-                *instr = _Py_MAKECODEUNIT(BINARY_OP_SUBTRACT_INT,
-                                          _Py_OPARG(*instr));
+                *instr = _Py_MAKECODEUNIT(BINARY_OP_SUBTRACT_INT, oparg);
                 goto success;
             }
             if (PyFloat_CheckExact(lhs)) {
-                *instr = _Py_MAKECODEUNIT(BINARY_OP_SUBTRACT_FLOAT,
-                                          _Py_OPARG(*instr));
+                *instr = _Py_MAKECODEUNIT(BINARY_OP_SUBTRACT_FLOAT, oparg);
                 goto success;
             }
             break;
@@ -1990,18 +1990,17 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
             // back to BINARY_OP (unless we're collecting stats, where it's more
             // important to get accurate hit counts for the unadaptive version
             // and each of the different failure types):
-            *instr = _Py_MAKECODEUNIT(BINARY_OP, adaptive->original_oparg);
+            *instr = _Py_MAKECODEUNIT(BINARY_OP, oparg);
             return;
 #endif
     }
-    SPECIALIZATION_FAIL(
-        BINARY_OP, binary_op_fail_kind(adaptive->original_oparg, lhs, rhs));
+    SPECIALIZATION_FAIL(BINARY_OP, binary_op_fail_kind(oparg, lhs, rhs));
     STAT_INC(BINARY_OP, failure);
-    cache_backoff(adaptive);
+    cache->counter = ADAPTIVE_CACHE_BACKOFF;
     return;
 success:
     STAT_INC(BINARY_OP, success);
-    adaptive->counter = initial_counter_value();
+    cache->counter = initial_counter_value();
 }
 
 
diff --git a/Python/wordcode_helpers.h b/Python/wordcode_helpers.h
deleted file mode 100644
index c8f7a0f41f9..00000000000
--- a/Python/wordcode_helpers.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/* This file contains code shared by the compiler and the peephole
-   optimizer.
- */
-
-#ifdef WORDS_BIGENDIAN
-#  define PACKOPARG(opcode, oparg) ((_Py_CODEUNIT)(((opcode) << 8) | (oparg)))
-#else
-#  define PACKOPARG(opcode, oparg) ((_Py_CODEUNIT)(((oparg) << 8) | (opcode)))
-#endif
-
-/* Minimum number of code units necessary to encode instruction with
-   EXTENDED_ARGs */
-static int
-instrsize(unsigned int oparg)
-{
-    return oparg <= 0xff ? 1 :
-        oparg <= 0xffff ? 2 :
-        oparg <= 0xffffff ? 3 :
-        4;
-}
-
-/* Spits out op/oparg pair using ilen bytes. codestr should be pointed at the
-   desired location of the first EXTENDED_ARG */
-static void
-write_op_arg(_Py_CODEUNIT *codestr, unsigned char opcode,
-    unsigned int oparg, int ilen)
-{
-    switch (ilen) {
-        case 4:
-            *codestr++ = PACKOPARG(EXTENDED_ARG, (oparg >> 24) & 0xff);
-            /* fall through */
-        case 3:
-            *codestr++ = PACKOPARG(EXTENDED_ARG, (oparg >> 16) & 0xff);
-            /* fall through */
-        case 2:
-            *codestr++ = PACKOPARG(EXTENDED_ARG, (oparg >> 8) & 0xff);
-            /* fall through */
-        case 1:
-            *codestr++ = PACKOPARG(opcode, oparg & 0xff);
-            break;
-        default:
-            Py_UNREACHABLE();
-    }
-}