diff options
Diffstat (limited to 'Python')
| -rw-r--r-- | Python/ceval.c | 28 | ||||
| -rw-r--r-- | Python/clinic/sysmodule.c.h | 29 | ||||
| -rw-r--r-- | Python/specialize.c | 197 | ||||
| -rw-r--r-- | Python/sysmodule.c | 14 |
4 files changed, 261 insertions, 7 deletions
diff --git a/Python/ceval.c b/Python/ceval.c index 4dff7bd2df9..619eff29e18 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1343,6 +1343,14 @@ eval_frame_handle_pending(PyThreadState *tstate) #define JUMPTO(x) (next_instr = first_instr + (x)) #define JUMPBY(x) (next_instr += (x)) +/* Get opcode and oparg from original instructions, not quickened form. */ +#define TRACING_NEXTOPARG() do { \ + _Py_CODEUNIT word = ((_Py_CODEUNIT *)PyBytes_AS_STRING(co->co_code))[INSTR_OFFSET()]; \ + opcode = _Py_OPCODE(word); \ + oparg = _Py_OPARG(word); \ + next_instr++; \ + } while (0) + /* OpCode prediction macros Some opcodes tend to come in pairs thus making it possible to predict the second code when the first is run. For example, @@ -1644,15 +1652,23 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) if (PyDTrace_FUNCTION_ENTRY_ENABLED()) dtrace_function_entry(f); + /* Increment the warmup counter and quicken if warm enough + * _Py_Quicken is idempotent so we don't worry about overflow */ + if (!PyCodeObject_IsWarmedUp(co)) { + PyCodeObject_IncrementWarmup(co); + if (PyCodeObject_IsWarmedUp(co)) { + if (_Py_Quicken(co)) { + goto exit_eval_frame; + } + } + } + + names = co->co_names; consts = co->co_consts; fastlocals = f->f_localsptr; + first_instr = co->co_firstinstr; freevars = f->f_localsptr + co->co_nlocals; - assert(PyBytes_Check(co->co_code)); - assert(PyBytes_GET_SIZE(co->co_code) <= INT_MAX); - assert(PyBytes_GET_SIZE(co->co_code) % sizeof(_Py_CODEUNIT) == 0); - assert(_Py_IS_ALIGNED(PyBytes_AS_STRING(co->co_code), sizeof(_Py_CODEUNIT))); - first_instr = (_Py_CODEUNIT *) PyBytes_AS_STRING(co->co_code); /* f->f_lasti refers to the index of the last instruction, unless it's -1 in which case next_instr should be first_instr. @@ -1757,7 +1773,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) tracing_dispatch: f->f_lasti = INSTR_OFFSET(); - NEXTOPARG(); + TRACING_NEXTOPARG(); if (PyDTrace_LINE_ENABLED()) maybe_dtrace_line(f, &trace_info); diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 04c84811498..763fe7a96fe 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -710,6 +710,33 @@ exit: #endif /* defined(Py_REF_DEBUG) */ +PyDoc_STRVAR(sys__getquickenedcount__doc__, +"_getquickenedcount($module, /)\n" +"--\n" +"\n"); + +#define SYS__GETQUICKENEDCOUNT_METHODDEF \ + {"_getquickenedcount", (PyCFunction)sys__getquickenedcount, METH_NOARGS, sys__getquickenedcount__doc__}, + +static Py_ssize_t +sys__getquickenedcount_impl(PyObject *module); + +static PyObject * +sys__getquickenedcount(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + Py_ssize_t _return_value; + + _return_value = sys__getquickenedcount_impl(module); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(sys_getallocatedblocks__doc__, "getallocatedblocks($module, /)\n" "--\n" @@ -983,4 +1010,4 @@ sys__deactivate_opcache(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=68c62b9ca317a0c8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e77bf636a177c5c3 input=a9049054013a1b77]*/ diff --git a/Python/specialize.c b/Python/specialize.c new file mode 100644 index 00000000000..07152d80538 --- /dev/null +++ b/Python/specialize.c @@ -0,0 +1,197 @@ + +#include "Python.h" +#include "pycore_code.h" +#include "opcode.h" + + +/* We layout the quickened data as a bi-directional array: + * Instructions upwards, cache entries downwards. + * first_instr is aligned to a SpecializedCacheEntry. + * The nth instruction is located at first_instr[n] + * The nth cache is located at ((SpecializedCacheEntry *)first_instr)[-1-n] + * The first (index 0) cache entry is reserved for the count, to enable finding + * the first instruction from the base pointer. + * The cache_count argument must include space for the count. + * We use the SpecializedCacheOrInstruction union to refer to the data + * to avoid type punning. + + Layout of quickened data, each line 8 bytes for M cache entries and N instructions: + + <cache_count> <---- co->co_quickened + <cache M-1> + <cache M-2> + ... + <cache 0> + <instr 0> <instr 1> <instr 2> <instr 3> <--- co->co_first_instr + <instr 4> <instr 5> <instr 6> <instr 7> + ... + <instr N-1> +*/ + +Py_ssize_t _Py_QuickenedCount = 0; + +static SpecializedCacheOrInstruction * +allocate(int cache_count, int instruction_count) +{ + assert(sizeof(SpecializedCacheOrInstruction) == 2*sizeof(int32_t)); + assert(sizeof(SpecializedCacheEntry) == 2*sizeof(int32_t)); + assert(cache_count > 0); + assert(instruction_count > 0); + int count = cache_count + (instruction_count + INSTRUCTIONS_PER_ENTRY -1)/INSTRUCTIONS_PER_ENTRY; + SpecializedCacheOrInstruction *array = (SpecializedCacheOrInstruction *) + PyMem_Malloc(sizeof(SpecializedCacheOrInstruction) * count); + if (array == NULL) { + PyErr_NoMemory(); + return NULL; + } + _Py_QuickenedCount++; + array[0].entry.zero.cache_count = cache_count; + return array; +} + +static int +get_cache_count(SpecializedCacheOrInstruction *quickened) { + return quickened[0].entry.zero.cache_count; +} + +/* Map from opcode to adaptive opcode. + Values of zero are ignored. */ +static uint8_t adaptive_opcodes[256] = { 0 }; + +/* The number of cache entries required for a "family" of instructions. */ +static uint8_t cache_requirements[256] = { 0 }; + +/* Return the oparg for the cache_offset and instruction index. + * + * If no cache is needed then return the original oparg. + * If a cache is needed, but cannot be accessed because + * oparg would be too large, then return -1. + * + * Also updates the cache_offset, as it may need to be incremented by + * more than the cache requirements, if many instructions do not need caches. + * + * See pycore_code.h for details of how the cache offset, + * instruction index and oparg are related */ +static int +oparg_from_instruction_and_update_offset(int index, int opcode, int original_oparg, int *cache_offset) { + /* The instruction pointer in the interpreter points to the next + * instruction, so we compute the offset using nexti (index + 1) */ + int nexti = index + 1; + uint8_t need = cache_requirements[opcode]; + if (need == 0) { + return original_oparg; + } + assert(adaptive_opcodes[opcode] != 0); + int oparg = oparg_from_offset_and_nexti(*cache_offset, nexti); + assert(*cache_offset == offset_from_oparg_and_nexti(oparg, nexti)); + /* Some cache space is wasted here as the minimum possible offset is (nexti>>1) */ + if (oparg < 0) { + oparg = 0; + *cache_offset = offset_from_oparg_and_nexti(oparg, nexti); + } + else if (oparg > 255) { + return -1; + } + *cache_offset += need; + return oparg; +} + +static int +entries_needed(_Py_CODEUNIT *code, int len) +{ + int cache_offset = 0; + int previous_opcode = -1; + for (int i = 0; i < len; i++) { + uint8_t opcode = _Py_OPCODE(code[i]); + if (previous_opcode != EXTENDED_ARG) { + oparg_from_instruction_and_update_offset(i, opcode, 0, &cache_offset); + } + previous_opcode = opcode; + } + return cache_offset + 1; // One extra for the count entry +} + +static inline _Py_CODEUNIT * +first_instruction(SpecializedCacheOrInstruction *quickened) +{ + return &quickened[get_cache_count(quickened)].code[0]; +} + +/** Insert adaptive instructions and superinstructions. + * + * Skip instruction preceded by EXTENDED_ARG for adaptive + * instructions as those are both very rare and tricky + * to handle. + */ +static void +optimize(SpecializedCacheOrInstruction *quickened, int len) +{ + _Py_CODEUNIT *instructions = first_instruction(quickened); + int cache_offset = 0; + int previous_opcode = -1; + for(int i = 0; i < len; i++) { + int opcode = _Py_OPCODE(instructions[i]); + int oparg = _Py_OPARG(instructions[i]); + uint8_t adaptive_opcode = adaptive_opcodes[opcode]; + if (adaptive_opcode && previous_opcode != EXTENDED_ARG) { + int new_oparg = oparg_from_instruction_and_update_offset( + i, opcode, oparg, &cache_offset + ); + if (new_oparg < 0) { + /* Not possible to allocate a cache for this instruction */ + previous_opcode = opcode; + continue; + } + instructions[i] = _Py_MAKECODEUNIT(adaptive_opcode, new_oparg); + previous_opcode = adaptive_opcode; + int entries_needed = cache_requirements[opcode]; + if (entries_needed) { + /* Initialize the adpative cache entry */ + int cache0_offset = cache_offset-entries_needed; + SpecializedCacheEntry *cache = + _GetSpecializedCacheEntry(instructions, cache0_offset); + cache->adaptive.original_oparg = oparg; + cache->adaptive.counter = 0; + } + } + else { + /* Super instructions don't use the cache, + * so no need to update the offset. */ + switch (opcode) { + /* Insert superinstructions here + E.g. + case LOAD_FAST: + if (previous_opcode == LOAD_FAST) + instructions[i-1] = _Py_MAKECODEUNIT(LOAD_FAST__LOAD_FAST, oparg); + */ + } + previous_opcode = opcode; + } + } + assert(cache_offset+1 == get_cache_count(quickened)); +} + +int +_Py_Quicken(PyCodeObject *code) { + if (code->co_quickened) { + return 0; + } + Py_ssize_t size = PyBytes_GET_SIZE(code->co_code); + int instr_count = (int)(size/sizeof(_Py_CODEUNIT)); + if (instr_count > MAX_SIZE_TO_QUICKEN) { + code->co_warmup = QUICKENING_WARMUP_COLDEST; + return 0; + } + int entry_count = entries_needed(code->co_firstinstr, instr_count); + SpecializedCacheOrInstruction *quickened = allocate(entry_count, instr_count); + if (quickened == NULL) { + return -1; + } + _Py_CODEUNIT *new_instructions = first_instruction(quickened); + memcpy(new_instructions, code->co_firstinstr, size); + optimize(quickened, instr_count); + code->co_quickened = quickened; + code->co_firstinstr = new_instructions; + return 0; +} + diff --git a/Python/sysmodule.c b/Python/sysmodule.c index f1f4492f83c..b71a096eaa0 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -18,6 +18,7 @@ Data members: #include "pycore_ceval.h" // _Py_RecursionLimitLowerWaterMark() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() #include "pycore_object.h" // _PyObject_IS_GC() +#include "pycore_code.h" // _Py_QuickenedCount #include "pycore_pathconfig.h" // _PyPathConfig_ComputeSysPath0() #include "pycore_pyerrors.h" // _PyErr_Fetch() #include "pycore_pylifecycle.h" // _PyErr_WriteUnraisableDefaultHook() @@ -1763,9 +1764,21 @@ sys_gettotalrefcount_impl(PyObject *module) { return _Py_GetRefTotal(); } + #endif /* Py_REF_DEBUG */ /*[clinic input] +sys._getquickenedcount -> Py_ssize_t +[clinic start generated code]*/ + +static Py_ssize_t +sys__getquickenedcount_impl(PyObject *module) +/*[clinic end generated code: output=1ab259e7f91248a2 input=249d448159eca912]*/ +{ + return _Py_QuickenedCount; +} + +/*[clinic input] sys.getallocatedblocks -> Py_ssize_t Return the number of memory blocks currently allocated. @@ -1995,6 +2008,7 @@ static PyMethodDef sys_methods[] = { #endif SYS_GETFILESYSTEMENCODING_METHODDEF SYS_GETFILESYSTEMENCODEERRORS_METHODDEF + SYS__GETQUICKENEDCOUNT_METHODDEF #ifdef Py_TRACE_REFS {"getobjects", _Py_GetObjects, METH_VARARGS}, #endif |