diff options
Diffstat (limited to 'Python')
| -rw-r--r-- | Python/bltinmodule.c | 163 | ||||
| -rw-r--r-- | Python/bytecodes.c | 52 | ||||
| -rw-r--r-- | Python/ceval.c | 119 | ||||
| -rw-r--r-- | Python/crossinterp.c | 452 | ||||
| -rw-r--r-- | Python/crossinterp_data_lookup.h | 24 | ||||
| -rw-r--r-- | Python/emscripten_trampoline.c | 2 | ||||
| -rw-r--r-- | Python/executor_cases.c.h | 142 | ||||
| -rw-r--r-- | Python/generated_cases.c.h | 2 | ||||
| -rw-r--r-- | Python/getversion.c | 2 | ||||
| -rw-r--r-- | Python/import.c | 15 | ||||
| -rw-r--r-- | Python/optimizer.c | 4 | ||||
| -rw-r--r-- | Python/optimizer_analysis.c | 15 | ||||
| -rw-r--r-- | Python/optimizer_bytecodes.c | 76 | ||||
| -rw-r--r-- | Python/optimizer_cases.c.h | 602 | ||||
| -rw-r--r-- | Python/optimizer_symbols.c | 332 | ||||
| -rw-r--r-- | Python/pystate.c | 1 | ||||
| -rw-r--r-- | Python/specialize.c | 1 |
17 files changed, 1241 insertions, 763 deletions
diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index e08c63924ca..51d7297ec24 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -957,6 +957,7 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, PyObject *locals) /*[clinic end generated code: output=0a0824aa70093116 input=7c7bce5299a89062]*/ { + PyThreadState *tstate = _PyThreadState_GET(); PyObject *result = NULL, *source_copy; const char *str; @@ -970,35 +971,46 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, : "globals must be a dict"); return NULL; } - if (globals == Py_None) { + + int fromframe = 0; + if (globals != Py_None) { + Py_INCREF(globals); + } + else if (_PyEval_GetFrame() != NULL) { + fromframe = 1; globals = PyEval_GetGlobals(); - if (locals == Py_None) { - locals = _PyEval_GetFrameLocals(); - if (locals == NULL) - return NULL; - } - else { - Py_INCREF(locals); - } + assert(globals != NULL); + Py_INCREF(globals); } - else if (locals == Py_None) - locals = Py_NewRef(globals); else { - Py_INCREF(locals); + globals = _PyEval_GetGlobalsFromRunningMain(tstate); + if (globals == NULL) { + if (!_PyErr_Occurred(tstate)) { + PyErr_SetString(PyExc_TypeError, + "eval must be given globals and locals " + "when called without a frame"); + } + return NULL; + } + Py_INCREF(globals); } - if (globals == NULL || locals == NULL) { - PyErr_SetString(PyExc_TypeError, - "eval must be given globals and locals " - "when called without a frame"); - goto error; + if (locals != Py_None) { + Py_INCREF(locals); } - - int r = PyDict_Contains(globals, &_Py_ID(__builtins__)); - if (r == 0) { - r = PyDict_SetItem(globals, &_Py_ID(__builtins__), PyEval_GetBuiltins()); + else if (fromframe) { + locals = _PyEval_GetFrameLocals(); + if (locals == NULL) { + assert(PyErr_Occurred()); + Py_DECREF(globals); + return NULL; + } + } + else { + locals = Py_NewRef(globals); } - if (r < 0) { + + if (_PyEval_EnsureBuiltins(tstate, globals, NULL) < 0) { goto error; } @@ -1039,6 +1051,7 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, } error: + Py_XDECREF(globals); Py_XDECREF(locals); return result; } @@ -1069,29 +1082,44 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, PyObject *locals, PyObject *closure) /*[clinic end generated code: output=7579eb4e7646743d input=25e989b6d87a3a21]*/ { + PyThreadState *tstate = _PyThreadState_GET(); PyObject *v; - if (globals == Py_None) { + int fromframe = 0; + if (globals != Py_None) { + Py_INCREF(globals); + } + else if (_PyEval_GetFrame() != NULL) { + fromframe = 1; globals = PyEval_GetGlobals(); - if (locals == Py_None) { - locals = _PyEval_GetFrameLocals(); - if (locals == NULL) - return NULL; - } - else { - Py_INCREF(locals); + assert(globals != NULL); + Py_INCREF(globals); + } + else { + globals = _PyEval_GetGlobalsFromRunningMain(tstate); + if (globals == NULL) { + if (!_PyErr_Occurred(tstate)) { + PyErr_SetString(PyExc_SystemError, + "globals and locals cannot be NULL"); + } + goto error; } - if (!globals || !locals) { - PyErr_SetString(PyExc_SystemError, - "globals and locals cannot be NULL"); + Py_INCREF(globals); + } + + if (locals != Py_None) { + Py_INCREF(locals); + } + else if (fromframe) { + locals = _PyEval_GetFrameLocals(); + if (locals == NULL) { + assert(PyErr_Occurred()); + Py_DECREF(globals); return NULL; } } - else if (locals == Py_None) { - locals = Py_NewRef(globals); - } else { - Py_INCREF(locals); + locals = Py_NewRef(globals); } if (!PyDict_Check(globals)) { @@ -1105,11 +1133,8 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, Py_TYPE(locals)->tp_name); goto error; } - int r = PyDict_Contains(globals, &_Py_ID(__builtins__)); - if (r == 0) { - r = PyDict_SetItem(globals, &_Py_ID(__builtins__), PyEval_GetBuiltins()); - } - if (r < 0) { + + if (_PyEval_EnsureBuiltins(tstate, globals, NULL) < 0) { goto error; } @@ -1186,11 +1211,13 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, } if (v == NULL) goto error; + Py_DECREF(globals); Py_DECREF(locals); Py_DECREF(v); Py_RETURN_NONE; error: + Py_XDECREF(globals); Py_XDECREF(locals); return NULL; } @@ -1240,10 +1267,21 @@ static PyObject * builtin_globals_impl(PyObject *module) /*[clinic end generated code: output=e5dd1527067b94d2 input=9327576f92bb48ba]*/ { - PyObject *d; - - d = PyEval_GetGlobals(); - return Py_XNewRef(d); + PyObject *globals; + if (_PyEval_GetFrame() != NULL) { + globals = PyEval_GetGlobals(); + assert(globals != NULL); + return Py_NewRef(globals); + } + PyThreadState *tstate = _PyThreadState_GET(); + globals = _PyEval_GetGlobalsFromRunningMain(tstate); + if (globals == NULL) { + if (_PyErr_Occurred(tstate)) { + return NULL; + } + Py_RETURN_NONE; + } + return Py_NewRef(globals); } @@ -1887,7 +1925,21 @@ static PyObject * builtin_locals_impl(PyObject *module) /*[clinic end generated code: output=b46c94015ce11448 input=7874018d478d5c4b]*/ { - return _PyEval_GetFrameLocals(); + PyObject *locals; + if (_PyEval_GetFrame() != NULL) { + locals = _PyEval_GetFrameLocals(); + assert(locals != NULL || PyErr_Occurred()); + return locals; + } + PyThreadState *tstate = _PyThreadState_GET(); + locals = _PyEval_GetGlobalsFromRunningMain(tstate); + if (locals == NULL) { + if (_PyErr_Occurred(tstate)) { + return NULL; + } + Py_RETURN_NONE; + } + return Py_NewRef(locals); } @@ -2623,7 +2675,22 @@ builtin_vars(PyObject *self, PyObject *args) if (!PyArg_UnpackTuple(args, "vars", 0, 1, &v)) return NULL; if (v == NULL) { - d = _PyEval_GetFrameLocals(); + if (_PyEval_GetFrame() != NULL) { + d = _PyEval_GetFrameLocals(); + } + else { + PyThreadState *tstate = _PyThreadState_GET(); + d = _PyEval_GetGlobalsFromRunningMain(tstate); + if (d == NULL) { + if (!_PyErr_Occurred(tstate)) { + d = _PyEval_GetFrameLocals(); + assert(_PyErr_Occurred(tstate)); + } + } + else { + Py_INCREF(d); + } + } } else { if (PyObject_GetOptionalAttr(v, &_Py_ID(__dict__), &d) == 0) { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 971e97a5784..2dd1d27747a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -687,6 +687,52 @@ dummy_func( ERROR_IF(PyStackRef_IsNull(res)); } + + pure op(_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsNull(res)); + } + + pure op(_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsNull(res)); + } + + pure op(_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsNull(res)); + } + macro(BINARY_OP_MULTIPLY_FLOAT) = _GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/5 + _BINARY_OP_MULTIPLY_FLOAT; macro(BINARY_OP_ADD_FLOAT) = @@ -4946,8 +4992,7 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - pure inst(COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { - assert(oparg > 0); + pure replicate(1:4) inst(COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { top = PyStackRef_DUP(bottom); } @@ -4980,12 +5025,11 @@ dummy_func( macro(BINARY_OP) = _SPECIALIZE_BINARY_OP + unused/4 + _BINARY_OP; - pure inst(SWAP, (bottom, unused[oparg-2], top -- + pure replicate(2:4) inst(SWAP, (bottom, unused[oparg-2], top -- bottom, unused[oparg-2], top)) { _PyStackRef temp = bottom; bottom = top; top = temp; - assert(oparg >= 2); } inst(INSTRUMENTED_LINE, ( -- )) { diff --git a/Python/ceval.c b/Python/ceval.c index 4cfe4bb88f4..3da6f61f5ac 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2746,10 +2746,9 @@ _PyEval_GetFrameLocals(void) return locals; } -PyObject * -PyEval_GetGlobals(void) +static PyObject * +_PyEval_GetGlobals(PyThreadState *tstate) { - PyThreadState *tstate = _PyThreadState_GET(); _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); if (current_frame == NULL) { return NULL; @@ -2757,6 +2756,120 @@ PyEval_GetGlobals(void) return current_frame->f_globals; } +PyObject * +PyEval_GetGlobals(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + return _PyEval_GetGlobals(tstate); +} + +PyObject * +_PyEval_GetGlobalsFromRunningMain(PyThreadState *tstate) +{ + if (!_PyInterpreterState_IsRunningMain(tstate->interp)) { + return NULL; + } + PyObject *mod = _Py_GetMainModule(tstate); + if (_Py_CheckMainModule(mod) < 0) { + Py_XDECREF(mod); + return NULL; + } + PyObject *globals = PyModule_GetDict(mod); // borrowed + Py_DECREF(mod); + return globals; +} + +static PyObject * +get_globals_builtins(PyObject *globals) +{ + PyObject *builtins = NULL; + if (PyDict_Check(globals)) { + if (PyDict_GetItemRef(globals, &_Py_ID(__builtins__), &builtins) < 0) { + return NULL; + } + } + else { + if (PyMapping_GetOptionalItem( + globals, &_Py_ID(__builtins__), &builtins) < 0) + { + return NULL; + } + } + return builtins; +} + +static int +set_globals_builtins(PyObject *globals, PyObject *builtins) +{ + if (PyDict_Check(globals)) { + if (PyDict_SetItem(globals, &_Py_ID(__builtins__), builtins) < 0) { + return -1; + } + } + else { + if (PyObject_SetItem(globals, &_Py_ID(__builtins__), builtins) < 0) { + return -1; + } + } + return 0; +} + +int +_PyEval_EnsureBuiltins(PyThreadState *tstate, PyObject *globals, + PyObject **p_builtins) +{ + PyObject *builtins = get_globals_builtins(globals); + if (builtins == NULL) { + if (_PyErr_Occurred(tstate)) { + return -1; + } + builtins = PyEval_GetBuiltins(); // borrowed + if (builtins == NULL) { + assert(_PyErr_Occurred(tstate)); + return -1; + } + Py_INCREF(builtins); + if (set_globals_builtins(globals, builtins) < 0) { + Py_DECREF(builtins); + return -1; + } + } + if (p_builtins != NULL) { + *p_builtins = builtins; + } + else { + Py_DECREF(builtins); + } + return 0; +} + +int +_PyEval_EnsureBuiltinsWithModule(PyThreadState *tstate, PyObject *globals, + PyObject **p_builtins) +{ + PyObject *builtins = get_globals_builtins(globals); + if (builtins == NULL) { + if (_PyErr_Occurred(tstate)) { + return -1; + } + builtins = PyImport_ImportModuleLevel("builtins", NULL, NULL, NULL, 0); + if (builtins == NULL) { + return -1; + } + if (set_globals_builtins(globals, builtins) < 0) { + Py_DECREF(builtins); + return -1; + } + } + if (p_builtins != NULL) { + *p_builtins = builtins; + } + else { + Py_DECREF(builtins); + } + return 0; +} + PyObject* PyEval_GetFrameLocals(void) { diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 39c7ea69890..aa958fe2f5c 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -7,9 +7,12 @@ #include "pycore_ceval.h" // _Py_simple_func #include "pycore_crossinterp.h" // _PyXIData_t #include "pycore_function.h" // _PyFunction_VerifyStateless() +#include "pycore_global_strings.h" // _Py_ID() +#include "pycore_import.h" // _PyImport_SetModule() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_pythonrun.h" // _Py_SourceAsString() +#include "pycore_runtime.h" // _PyRuntime #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_typeobject.h" // _PyStaticType_InitBuiltin() @@ -22,6 +25,7 @@ _Py_GetMainfile(char *buffer, size_t maxlen) PyThreadState *tstate = _PyThreadState_GET(); PyObject *module = _Py_GetMainModule(tstate); if (_Py_CheckMainModule(module) < 0) { + Py_XDECREF(module); return -1; } Py_ssize_t size = _PyModule_GetFilenameUTF8(module, buffer, maxlen); @@ -31,27 +35,6 @@ _Py_GetMainfile(char *buffer, size_t maxlen) static PyObject * -import_get_module(PyThreadState *tstate, const char *modname) -{ - PyObject *module = NULL; - if (strcmp(modname, "__main__") == 0) { - module = _Py_GetMainModule(tstate); - if (_Py_CheckMainModule(module) < 0) { - assert(_PyErr_Occurred(tstate)); - return NULL; - } - } - else { - module = PyImport_ImportModule(modname); - if (module == NULL) { - return NULL; - } - } - return module; -} - - -static PyObject * runpy_run_path(const char *filename, const char *modname) { PyObject *run_path = PyImport_ImportModuleAttrString("runpy", "run_path"); @@ -81,97 +64,181 @@ set_exc_with_cause(PyObject *exctype, const char *msg) } -static PyObject * -pyerr_get_message(PyObject *exc) +/****************************/ +/* module duplication utils */ +/****************************/ + +struct sync_module_result { + PyObject *module; + PyObject *loaded; + PyObject *failed; +}; + +struct sync_module { + const char *filename; + char _filename[MAXPATHLEN+1]; + struct sync_module_result cached; +}; + +static void +sync_module_clear(struct sync_module *data) { - assert(!PyErr_Occurred()); - PyObject *args = PyException_GetArgs(exc); - if (args == NULL || args == Py_None || PyObject_Size(args) < 1) { - return NULL; - } - if (PyUnicode_Check(args)) { - return args; - } - PyObject *msg = PySequence_GetItem(args, 0); - Py_DECREF(args); - if (msg == NULL) { - PyErr_Clear(); - return NULL; - } - if (!PyUnicode_Check(msg)) { - Py_DECREF(msg); - return NULL; - } - return msg; + data->filename = NULL; + Py_CLEAR(data->cached.module); + Py_CLEAR(data->cached.loaded); + Py_CLEAR(data->cached.failed); } -#define MAX_MODNAME (255) -#define MAX_ATTRNAME (255) +static void +sync_module_capture_exc(PyThreadState *tstate, struct sync_module *data) +{ + assert(_PyErr_Occurred(tstate)); + PyObject *context = data->cached.failed; + PyObject *exc = _PyErr_GetRaisedException(tstate); + _PyErr_SetRaisedException(tstate, Py_NewRef(exc)); + if (context != NULL) { + PyException_SetContext(exc, context); + } + data->cached.failed = exc; +} -struct attributeerror_info { - char modname[MAX_MODNAME+1]; - char attrname[MAX_ATTRNAME+1]; -}; static int -_parse_attributeerror(PyObject *exc, struct attributeerror_info *info) +ensure_isolated_main(PyThreadState *tstate, struct sync_module *main) { - assert(exc != NULL); - assert(PyErr_GivenExceptionMatches(exc, PyExc_AttributeError)); - int res = -1; + // Load the module from the original file (or from a cache). - PyObject *msgobj = pyerr_get_message(exc); - if (msgobj == NULL) { + // First try the local cache. + if (main->cached.failed != NULL) { + // We'll deal with this in apply_isolated_main(). + assert(main->cached.module == NULL); + assert(main->cached.loaded == NULL); + return 0; + } + else if (main->cached.loaded != NULL) { + assert(main->cached.module != NULL); + return 0; + } + assert(main->cached.module == NULL); + + if (main->filename == NULL) { + _PyErr_SetString(tstate, PyExc_NotImplementedError, ""); return -1; } - const char *err = PyUnicode_AsUTF8(msgobj); - if (strncmp(err, "module '", 8) != 0) { - goto finally; + // It wasn't in the local cache so we'll need to populate it. + PyObject *mod = _Py_GetMainModule(tstate); + if (_Py_CheckMainModule(mod) < 0) { + // This is probably unrecoverable, so don't bother caching the error. + assert(_PyErr_Occurred(tstate)); + Py_XDECREF(mod); + return -1; } - err += 8; + PyObject *loaded = NULL; - const char *matched = strchr(err, '\''); - if (matched == NULL) { - goto finally; + // Try the per-interpreter cache for the loaded module. + // XXX Store it in sys.modules? + PyObject *interpns = PyInterpreterState_GetDict(tstate->interp); + assert(interpns != NULL); + PyObject *key = PyUnicode_FromString("CACHED_MODULE_NS___main__"); + if (key == NULL) { + // It's probably unrecoverable, so don't bother caching the error. + Py_DECREF(mod); + return -1; } - Py_ssize_t len = matched - err; - if (len > MAX_MODNAME) { - goto finally; + else if (PyDict_GetItemRef(interpns, key, &loaded) < 0) { + // It's probably unrecoverable, so don't bother caching the error. + Py_DECREF(mod); + Py_DECREF(key); + return -1; } - (void)strncpy(info->modname, err, len); - info->modname[len] = '\0'; - err = matched; + else if (loaded == NULL) { + // It wasn't already loaded from file. + loaded = PyModule_NewObject(&_Py_ID(__main__)); + if (loaded == NULL) { + goto error; + } + PyObject *ns = _PyModule_GetDict(loaded); - if (strncmp(err, "' has no attribute '", 20) != 0) { - goto finally; - } - err += 20; + // We don't want to trigger "if __name__ == '__main__':", + // so we use a bogus module name. + PyObject *loaded_ns = + runpy_run_path(main->filename, "<fake __main__>"); + if (loaded_ns == NULL) { + goto error; + } + int res = PyDict_Update(ns, loaded_ns); + Py_DECREF(loaded_ns); + if (res < 0) { + goto error; + } - matched = strchr(err, '\''); - if (matched == NULL) { - goto finally; - } - len = matched - err; - if (len > MAX_ATTRNAME) { - goto finally; + // Set the per-interpreter cache entry. + if (PyDict_SetItem(interpns, key, loaded) < 0) { + goto error; + } } - (void)strncpy(info->attrname, err, len); - info->attrname[len] = '\0'; - err = matched + 1; - if (strlen(err) > 0) { - goto finally; + Py_DECREF(key); + main->cached = (struct sync_module_result){ + .module = mod, + .loaded = loaded, + }; + return 0; + +error: + sync_module_capture_exc(tstate, main); + Py_XDECREF(loaded); + Py_DECREF(mod); + Py_XDECREF(key); + return -1; +} + +#ifndef NDEBUG +static int +main_mod_matches(PyObject *expected) +{ + PyObject *mod = PyImport_GetModule(&_Py_ID(__main__)); + Py_XDECREF(mod); + return mod == expected; +} +#endif + +static int +apply_isolated_main(PyThreadState *tstate, struct sync_module *main) +{ + assert((main->cached.loaded == NULL) == (main->cached.loaded == NULL)); + if (main->cached.failed != NULL) { + // It must have failed previously. + assert(main->cached.loaded == NULL); + _PyErr_SetRaisedException(tstate, main->cached.failed); + return -1; } - res = 0; + assert(main->cached.loaded != NULL); -finally: - Py_DECREF(msgobj); - return res; + assert(main_mod_matches(main->cached.module)); + if (_PyImport_SetModule(&_Py_ID(__main__), main->cached.loaded) < 0) { + sync_module_capture_exc(tstate, main); + return -1; + } + return 0; } -#undef MAX_MODNAME -#undef MAX_ATTRNAME +static void +restore_main(PyThreadState *tstate, struct sync_module *main) +{ + assert(main->cached.failed == NULL); + assert(main->cached.module != NULL); + assert(main->cached.loaded != NULL); + PyObject *exc = _PyErr_GetRaisedException(tstate); + assert(main_mod_matches(main->cached.loaded)); + int res = _PyImport_SetModule(&_Py_ID(__main__), main->cached.module); + assert(res == 0); + if (res < 0) { + PyErr_FormatUnraisable("Exception ignored while restoring __main__"); + } + _PyErr_SetRaisedException(tstate, exc); +} /**************/ @@ -518,28 +585,6 @@ _PyPickle_Dumps(struct _pickle_context *ctx, PyObject *obj) } -struct sync_module_result { - PyObject *module; - PyObject *loaded; - PyObject *failed; -}; - -struct sync_module { - const char *filename; - char _filename[MAXPATHLEN+1]; - struct sync_module_result cached; -}; - -static void -sync_module_clear(struct sync_module *data) -{ - data->filename = NULL; - Py_CLEAR(data->cached.module); - Py_CLEAR(data->cached.loaded); - Py_CLEAR(data->cached.failed); -} - - struct _unpickle_context { PyThreadState *tstate; // We only special-case the __main__ module, @@ -553,142 +598,86 @@ _unpickle_context_clear(struct _unpickle_context *ctx) sync_module_clear(&ctx->main); } -static struct sync_module_result -_unpickle_context_get_module(struct _unpickle_context *ctx, - const char *modname) +static int +check_missing___main___attr(PyObject *exc) { - if (strcmp(modname, "__main__") == 0) { - return ctx->main.cached; + assert(!PyErr_Occurred()); + if (!PyErr_GivenExceptionMatches(exc, PyExc_AttributeError)) { + return 0; } - else { - return (struct sync_module_result){ - .failed = PyExc_NotImplementedError, - }; + + // Get the error message. + PyObject *args = PyException_GetArgs(exc); + if (args == NULL || args == Py_None || PyObject_Size(args) < 1) { + assert(!PyErr_Occurred()); + return 0; } + PyObject *msgobj = args; + if (!PyUnicode_Check(msgobj)) { + msgobj = PySequence_GetItem(args, 0); + Py_DECREF(args); + if (msgobj == NULL) { + PyErr_Clear(); + return 0; + } + } + const char *err = PyUnicode_AsUTF8(msgobj); + + // Check if it's a missing __main__ attr. + int cmp = strncmp(err, "module '__main__' has no attribute '", 36); + Py_DECREF(msgobj); + return cmp == 0; } -static struct sync_module_result -_unpickle_context_set_module(struct _unpickle_context *ctx, - const char *modname) +static PyObject * +_PyPickle_Loads(struct _unpickle_context *ctx, PyObject *pickled) { - struct sync_module_result res = {0}; - struct sync_module_result *cached = NULL; - const char *filename = NULL; - const char *run_modname = modname; - if (strcmp(modname, "__main__") == 0) { - cached = &ctx->main.cached; - filename = ctx->main.filename; - // We don't want to trigger "if __name__ == '__main__':". - run_modname = "<fake __main__>"; - } - else { - res.failed = PyExc_NotImplementedError; - goto finally; - } + PyThreadState *tstate = ctx->tstate; - res.module = import_get_module(ctx->tstate, modname); - if (res.module == NULL) { - res.failed = _PyErr_GetRaisedException(ctx->tstate); - assert(res.failed != NULL); - goto finally; + PyObject *exc = NULL; + PyObject *loads = PyImport_ImportModuleAttrString("pickle", "loads"); + if (loads == NULL) { + return NULL; } - if (filename == NULL) { - Py_CLEAR(res.module); - res.failed = PyExc_NotImplementedError; + // Make an initial attempt to unpickle. + PyObject *obj = PyObject_CallOneArg(loads, pickled); + if (obj != NULL) { goto finally; } - res.loaded = runpy_run_path(filename, run_modname); - if (res.loaded == NULL) { - Py_CLEAR(res.module); - res.failed = _PyErr_GetRaisedException(ctx->tstate); - assert(res.failed != NULL); + assert(_PyErr_Occurred(tstate)); + if (ctx == NULL) { goto finally; } - -finally: - if (cached != NULL) { - assert(cached->module == NULL); - assert(cached->loaded == NULL); - assert(cached->failed == NULL); - *cached = res; - } - return res; -} - - -static int -_handle_unpickle_missing_attr(struct _unpickle_context *ctx, PyObject *exc) -{ - // The caller must check if an exception is set or not when -1 is returned. - assert(!_PyErr_Occurred(ctx->tstate)); - assert(PyErr_GivenExceptionMatches(exc, PyExc_AttributeError)); - struct attributeerror_info info; - if (_parse_attributeerror(exc, &info) < 0) { - return -1; + exc = _PyErr_GetRaisedException(tstate); + if (!check_missing___main___attr(exc)) { + goto finally; } - // Get the module. - struct sync_module_result mod = _unpickle_context_get_module(ctx, info.modname); - if (mod.failed != NULL) { - // It must have failed previously. - return -1; - } - if (mod.module == NULL) { - mod = _unpickle_context_set_module(ctx, info.modname); - if (mod.failed != NULL) { - return -1; - } - assert(mod.module != NULL); + // Temporarily swap in a fake __main__ loaded from the original + // file and cached. Note that functions will use the cached ns + // for __globals__, // not the actual module. + if (ensure_isolated_main(tstate, &ctx->main) < 0) { + goto finally; } - - // Bail out if it is unexpectedly set already. - if (PyObject_HasAttrString(mod.module, info.attrname)) { - return -1; + if (apply_isolated_main(tstate, &ctx->main) < 0) { + goto finally; } - // Try setting the attribute. - PyObject *value = NULL; - if (PyDict_GetItemStringRef(mod.loaded, info.attrname, &value) <= 0) { - return -1; - } - assert(value != NULL); - int res = PyObject_SetAttrString(mod.module, info.attrname, value); - Py_DECREF(value); - if (res < 0) { - return -1; + // Try to unpickle once more. + obj = PyObject_CallOneArg(loads, pickled); + restore_main(tstate, &ctx->main); + if (obj == NULL) { + goto finally; } + Py_CLEAR(exc); - return 0; -} - -static PyObject * -_PyPickle_Loads(struct _unpickle_context *ctx, PyObject *pickled) -{ - PyObject *loads = PyImport_ImportModuleAttrString("pickle", "loads"); - if (loads == NULL) { - return NULL; - } - PyObject *obj = PyObject_CallOneArg(loads, pickled); - if (ctx != NULL) { - while (obj == NULL) { - assert(_PyErr_Occurred(ctx->tstate)); - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - // We leave other failures unhandled. - break; - } - // Try setting the attr if not set. - PyObject *exc = _PyErr_GetRaisedException(ctx->tstate); - if (_handle_unpickle_missing_attr(ctx, exc) < 0) { - // Any resulting exceptions are ignored - // in favor of the original. - _PyErr_SetRaisedException(ctx->tstate, exc); - break; - } - Py_CLEAR(exc); - // Retry with the attribute set. - obj = PyObject_CallOneArg(loads, pickled); - } +finally: + if (exc != NULL) { + sync_module_capture_exc(tstate, &ctx->main); + // We restore the original exception. + // It might make sense to chain it (__context__). + _PyErr_SetRaisedException(tstate, exc); } Py_DECREF(loads); return obj; @@ -2619,7 +2608,9 @@ _PyXI_Enter(_PyXI_session *session, PyInterpreterState *interp, PyObject *nsupdates, _PyXI_session_result *result) { - PyThreadState *tstate = _PyThreadState_GET(); +#ifndef NDEBUG + PyThreadState *tstate = _PyThreadState_GET(); // Only used for asserts +#endif // Convert the attrs for cross-interpreter use. _PyXI_namespace *sharedns = NULL; @@ -2661,7 +2652,9 @@ _PyXI_Enter(_PyXI_session *session, _enter_session(session, interp); _PyXI_failure override = XI_FAILURE_INIT; override.code = _PyXI_ERR_UNCAUGHT_EXCEPTION; +#ifndef NDEBUG tstate = _PyThreadState_GET(); +#endif // Ensure this thread owns __main__. if (_PyInterpreterState_SetRunningMain(interp) < 0) { @@ -2697,7 +2690,9 @@ error: // Exit the session. _exit_session(session); +#ifndef NDEBUG tstate = _PyThreadState_GET(); +#endif if (sharedns != NULL) { _destroy_sharedns(sharedns); @@ -2883,6 +2878,7 @@ _ensure_main_ns(_PyXI_session *session, _PyXI_failure *failure) // Cache __main__.__dict__. PyObject *main_mod = _Py_GetMainModule(tstate); if (_Py_CheckMainModule(main_mod) < 0) { + Py_XDECREF(main_mod); if (failure != NULL) { *failure = (_PyXI_failure){ .code = _PyXI_ERR_MAIN_NS_FAILURE, diff --git a/Python/crossinterp_data_lookup.h b/Python/crossinterp_data_lookup.h index 6d0b93eb82a..c3c76ae8d9a 100644 --- a/Python/crossinterp_data_lookup.h +++ b/Python/crossinterp_data_lookup.h @@ -722,16 +722,26 @@ _PyFunction_FromXIData(_PyXIData_t *xidata) return NULL; } // Create a new function. + // For stateless functions (no globals) we use __main__ as __globals__, + // just like we do for builtins like exec(). assert(PyCode_Check(code)); - PyObject *globals = PyDict_New(); + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *globals = _PyEval_GetGlobalsFromRunningMain(tstate); // borrowed if (globals == NULL) { - Py_DECREF(code); - return NULL; + if (_PyErr_Occurred(tstate)) { + Py_DECREF(code); + return NULL; + } + globals = PyDict_New(); + if (globals == NULL) { + Py_DECREF(code); + return NULL; + } } - PyThreadState *tstate = _PyThreadState_GET(); - if (PyDict_SetItem(globals, &_Py_ID(__builtins__), - tstate->interp->builtins) < 0) - { + else { + Py_INCREF(globals); + } + if (_PyEval_EnsureBuiltins(tstate, globals, NULL) < 0) { Py_DECREF(code); Py_DECREF(globals); return NULL; diff --git a/Python/emscripten_trampoline.c b/Python/emscripten_trampoline.c index 975c28eec10..75b98a04723 100644 --- a/Python/emscripten_trampoline.c +++ b/Python/emscripten_trampoline.c @@ -80,7 +80,7 @@ function getPyEMCountArgsPtr() { // To differentiate, we check if the platform is 'MacIntel' (common for Macs and newer iPads) // AND if the device has multi-touch capabilities (navigator.maxTouchPoints > 1) (navigator.platform === 'MacIntel' && typeof navigator.maxTouchPoints !== 'undefined' && navigator.maxTouchPoints > 1) - ) + ); if (isIOS) { return 0; } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index dbfb2391bf0..3b58e7fab55 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1063,6 +1063,87 @@ break; } + case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + if (PyStackRef_IsNull(res)) { + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + if (PyStackRef_IsNull(res)) { + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + if (PyStackRef_IsNull(res)) { + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _BINARY_OP_ADD_UNICODE: { _PyStackRef right; _PyStackRef left; @@ -6763,12 +6844,44 @@ break; } + case _COPY_1: { + _PyStackRef bottom; + _PyStackRef top; + bottom = stack_pointer[-1]; + top = PyStackRef_DUP(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _COPY_2: { + _PyStackRef bottom; + _PyStackRef top; + bottom = stack_pointer[-2]; + top = PyStackRef_DUP(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _COPY_3: { + _PyStackRef bottom; + _PyStackRef top; + bottom = stack_pointer[-3]; + top = PyStackRef_DUP(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _COPY: { _PyStackRef bottom; _PyStackRef top; oparg = CURRENT_OPARG(); bottom = stack_pointer[-1 - (oparg-1)]; - assert(oparg > 0); top = PyStackRef_DUP(bottom); stack_pointer[0] = top; stack_pointer += 1; @@ -6808,6 +6921,32 @@ break; } + case _SWAP_2: { + _PyStackRef top; + _PyStackRef bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-2]; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + stack_pointer[-2] = bottom; + stack_pointer[-1] = top; + break; + } + + case _SWAP_3: { + _PyStackRef top; + _PyStackRef bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-3]; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + stack_pointer[-3] = bottom; + stack_pointer[-1] = top; + break; + } + case _SWAP: { _PyStackRef top; _PyStackRef bottom; @@ -6817,7 +6956,6 @@ _PyStackRef temp = bottom; bottom = top; top = temp; - assert(oparg >= 2); stack_pointer[-2 - (oparg-2)] = bottom; stack_pointer[-1] = top; break; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2cf027c539b..4fc1d5266d0 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5228,7 +5228,6 @@ _PyStackRef bottom; _PyStackRef top; bottom = stack_pointer[-1 - (oparg-1)]; - assert(oparg > 0); top = PyStackRef_DUP(bottom); stack_pointer[0] = top; stack_pointer += 1; @@ -11568,7 +11567,6 @@ _PyStackRef temp = bottom; bottom = top; top = temp; - assert(oparg >= 2); stack_pointer[-2 - (oparg-2)] = bottom; stack_pointer[-1] = top; DISPATCH(); diff --git a/Python/getversion.c b/Python/getversion.c index 226b2f999a6..8d8bc6ea700 100644 --- a/Python/getversion.c +++ b/Python/getversion.c @@ -15,7 +15,7 @@ void _Py_InitVersion(void) } initialized = 1; #ifdef Py_GIL_DISABLED - const char *buildinfo_format = "%.80s experimental free-threading build (%.80s) %.80s"; + const char *buildinfo_format = "%.80s free-threading build (%.80s) %.80s"; #else const char *buildinfo_format = "%.80s (%.80s) %.80s"; #endif diff --git a/Python/import.c b/Python/import.c index 184dede335d..73b94d0dd2a 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3960,25 +3960,28 @@ PyImport_Import(PyObject *module_name) } /* Get the builtins from current globals */ - globals = PyEval_GetGlobals(); + globals = PyEval_GetGlobals(); // borrowed if (globals != NULL) { Py_INCREF(globals); + // XXX Use _PyEval_EnsureBuiltins()? builtins = PyObject_GetItem(globals, &_Py_ID(__builtins__)); if (builtins == NULL) { // XXX Fall back to interp->builtins or sys.modules['builtins']? goto err; } } + else if (_PyErr_Occurred(tstate)) { + goto err; + } else { /* No globals -- use standard builtins, and fake globals */ - builtins = PyImport_ImportModuleLevel("builtins", - NULL, NULL, NULL, 0); - if (builtins == NULL) { + globals = PyDict_New(); + if (globals == NULL) { goto err; } - globals = Py_BuildValue("{OO}", &_Py_ID(__builtins__), builtins); - if (globals == NULL) + if (_PyEval_EnsureBuiltinsWithModule(tstate, globals, &builtins) < 0) { goto err; + } } /* Get the __import__ function from the builtins */ diff --git a/Python/optimizer.c b/Python/optimizer.c index dde3dd8ebe7..8d01d605ef4 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1292,8 +1292,8 @@ uop_optimize( for (int pc = 0; pc < length; pc++) { int opcode = buffer[pc].opcode; int oparg = buffer[pc].oparg; - if (oparg < _PyUop_Replication[opcode]) { - buffer[pc].opcode = opcode + oparg + 1; + if (oparg < _PyUop_Replication[opcode].stop && oparg >= _PyUop_Replication[opcode].start) { + buffer[pc].opcode = opcode + oparg + 1 - _PyUop_Replication[opcode].start; assert(strncmp(_PyOpcode_uop_name[buffer[pc].opcode], _PyOpcode_uop_name[opcode], strlen(_PyOpcode_uop_name[opcode])) == 0); } else if (is_terminator(&buffer[pc])) { diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 6a7df233819..de337f63716 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -347,8 +347,8 @@ static int optimize_to_bool( _PyUOpInstruction *this_instr, JitOptContext *ctx, - JitOptSymbol *value, - JitOptSymbol **result_ptr) + JitOptRef value, + JitOptRef *result_ptr) { if (sym_matches_type(value, &PyBool_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -375,7 +375,7 @@ eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit) } } -static JitOptSymbol * +static JitOptRef lookup_attr(JitOptContext *ctx, _PyUOpInstruction *this_instr, PyTypeObject *type, PyObject *name, uint16_t immortal, uint16_t mortal) @@ -440,6 +440,13 @@ get_code_with_logging(_PyUOpInstruction *op) return co; } +// TODO (gh-134584) generate most of this table automatically +const uint16_t op_without_decref_inputs[MAX_UOP_ID + 1] = { + [_BINARY_OP_MULTIPLY_FLOAT] = _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS, + [_BINARY_OP_ADD_FLOAT] = _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS, + [_BINARY_OP_SUBTRACT_FLOAT] = _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS, +}; + /* 1 for success, 0 for not ready, cannot error at the moment. */ static int optimize_uops( @@ -477,7 +484,7 @@ optimize_uops( int oparg = this_instr->oparg; opcode = this_instr->opcode; - JitOptSymbol **stack_pointer = ctx->frame->stack_pointer; + JitOptRef *stack_pointer = ctx->frame->stack_pointer; #ifdef Py_DEBUG if (get_lltrace() >= 3) { diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index c2469547d77..07b05fc3323 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -87,12 +87,12 @@ dummy_func(void) { } op(_LOAD_FAST_BORROW, (-- value)) { - value = GETLOCAL(oparg); + value = PyJitRef_Borrow(GETLOCAL(oparg)); } op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); - JitOptSymbol *temp = sym_new_null(ctx); + JitOptRef temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; } @@ -251,6 +251,10 @@ dummy_func(void) { else { res = sym_new_type(ctx, &PyFloat_Type); } + // TODO (gh-134584): Refactor this to use another uop + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } } op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { @@ -271,6 +275,10 @@ dummy_func(void) { else { res = sym_new_type(ctx, &PyFloat_Type); } + // TODO (gh-134584): Refactor this to use another uop + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } } op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { @@ -291,6 +299,10 @@ dummy_func(void) { else { res = sym_new_type(ctx, &PyFloat_Type); } + // TODO (gh-134584): Refactor this to use another uop + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } } op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { @@ -310,7 +322,7 @@ dummy_func(void) { } op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right -- )) { - JitOptSymbol *res; + JitOptRef res; if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); @@ -329,7 +341,7 @@ dummy_func(void) { } op(_BINARY_OP_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame)) { - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; } @@ -488,7 +500,7 @@ dummy_func(void) { op(_LOAD_CONST, (-- value)) { PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); } op(_LOAD_SMALL_INT, (-- value)) { @@ -496,35 +508,35 @@ dummy_func(void) { assert(val); assert(_Py_IsImmortal(val)); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); } op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_POP_TOP_LOAD_CONST_INLINE, (ptr/4, pop -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_POP_CALL_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused, unused -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused, unused, unused -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { @@ -533,7 +545,7 @@ dummy_func(void) { } op(_SWAP, (bottom, unused[oparg-2], top -- bottom, unused[oparg-2], top)) { - JitOptSymbol *temp = bottom; + JitOptRef temp = bottom; bottom = top; top = temp; assert(oparg >= 2); @@ -547,7 +559,7 @@ dummy_func(void) { op(_LOAD_ATTR_MODULE, (dict_version/2, index/1, owner -- attr)) { (void)dict_version; (void)index; - attr = NULL; + attr = PyJitRef_NULL; if (sym_is_const(ctx, owner)) { PyModuleObject *mod = (PyModuleObject *)sym_get_const(ctx, owner); if (PyModule_CheckExact(mod)) { @@ -561,7 +573,7 @@ dummy_func(void) { } } } - if (attr == NULL) { + if (PyJitRef_IsNull(attr)) { /* No conversion made. We don't know what `attr` is. */ attr = sym_new_not_null(ctx); } @@ -654,7 +666,7 @@ dummy_func(void) { op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame)) { (void)fget; - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; } @@ -672,6 +684,16 @@ dummy_func(void) { sym_set_type(callable, &PyFunction_Type); } + op(_CHECK_METHOD_VERSION, (func_version/2, callable, null, unused[oparg] -- callable, null, unused[oparg])) { + if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { + PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); + assert(PyMethod_Check(method)); + REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + this_instr->operand1 = (uintptr_t)method->im_func; + } + sym_set_type(callable, &PyMethod_Type); + } + op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { assert(sym_matches_type(callable, &PyFunction_Type)); if (sym_is_const(ctx, callable)) { @@ -702,7 +724,7 @@ dummy_func(void) { } - assert(self_or_null != NULL); + assert(!PyJitRef_IsNull(self_or_null)); assert(args != NULL); if (sym_is_not_null(self_or_null)) { // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM @@ -711,9 +733,9 @@ dummy_func(void) { } if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, args, argcount); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args, argcount)); } else { - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } } @@ -732,11 +754,11 @@ dummy_func(void) { break; } - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; } @@ -748,12 +770,12 @@ dummy_func(void) { } op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame)) { - init_frame = NULL; + init_frame = PyJitRef_NULL; ctx->done = true; } op(_RETURN_VALUE, (retval -- res)) { - JitOptSymbol *temp = retval; + JitOptRef temp = retval; DEAD(retval); SAVE_STACK(); ctx->frame->stack_pointer = stack_pointer; @@ -815,13 +837,13 @@ dummy_func(void) { } op(_FOR_ITER_GEN_FRAME, (unused, unused -- unused, unused, gen_frame)) { - gen_frame = NULL; + gen_frame = PyJitRef_NULL; /* We are about to hit the end of the trace */ ctx->done = true; } op(_SEND_GEN_FRAME, (unused, unused -- unused, gen_frame)) { - gen_frame = NULL; + gen_frame = PyJitRef_NULL; // We are about to hit the end of the trace: ctx->done = true; } @@ -841,7 +863,7 @@ dummy_func(void) { op(_PUSH_FRAME, (new_frame -- )) { SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; - ctx->frame = (_Py_UOpsAbstractFrame *)new_frame; + ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); ctx->curr_frame_depth++; stack_pointer = ctx->frame->stack_pointer; co = get_code(this_instr); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index d9313be0bb0..354331ef618 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -26,7 +26,7 @@ /* _MONITOR_RESUME is not a viable micro-op for tier 2 */ case _LOAD_FAST_CHECK: { - JitOptSymbol *value; + JitOptRef value; value = GETLOCAL(oparg); if (sym_is_null(value)) { ctx->done = true; @@ -38,7 +38,7 @@ } case _LOAD_FAST: { - JitOptSymbol *value; + JitOptRef value; value = GETLOCAL(oparg); stack_pointer[0] = value; stack_pointer += 1; @@ -47,8 +47,8 @@ } case _LOAD_FAST_BORROW: { - JitOptSymbol *value; - value = GETLOCAL(oparg); + JitOptRef value; + value = PyJitRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -56,9 +56,9 @@ } case _LOAD_FAST_AND_CLEAR: { - JitOptSymbol *value; + JitOptRef value; value = GETLOCAL(oparg); - JitOptSymbol *temp = sym_new_null(ctx); + JitOptRef temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; stack_pointer[0] = value; stack_pointer += 1; @@ -67,10 +67,10 @@ } case _LOAD_CONST: { - JitOptSymbol *value; + JitOptRef value; PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -78,12 +78,12 @@ } case _LOAD_SMALL_INT: { - JitOptSymbol *value; + JitOptRef value; PyObject *val = PyLong_FromLong(oparg); assert(val); assert(_Py_IsImmortal(val)); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -91,7 +91,7 @@ } case _STORE_FAST: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; GETLOCAL(oparg) = value; stack_pointer += -1; @@ -112,7 +112,7 @@ } case _PUSH_NULL: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -133,7 +133,7 @@ } case _END_SEND: { - JitOptSymbol *val; + JitOptRef val; val = sym_new_not_null(ctx); stack_pointer[-2] = val; stack_pointer += -1; @@ -142,15 +142,15 @@ } case _UNARY_NEGATIVE: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _UNARY_NOT: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; sym_set_type(value, &PyBool_Type); res = sym_new_truthiness(ctx, value, false); @@ -159,8 +159,8 @@ } case _TO_BOOL: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { @@ -171,7 +171,7 @@ } case _TO_BOOL_BOOL: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { @@ -183,8 +183,8 @@ } case _TO_BOOL_INT: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { @@ -196,7 +196,7 @@ } case _GUARD_NOS_LIST: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -206,7 +206,7 @@ } case _GUARD_TOS_LIST: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -220,8 +220,8 @@ } case _TO_BOOL_LIST: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { @@ -232,8 +232,8 @@ } case _TO_BOOL_NONE: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { @@ -245,7 +245,7 @@ } case _GUARD_NOS_UNICODE: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -255,7 +255,7 @@ } case _GUARD_TOS_UNICODE: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; if (sym_matches_type(value, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -265,8 +265,8 @@ } case _TO_BOOL_STR: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { @@ -277,7 +277,7 @@ } case _REPLACE_WITH_TRUE: { - JitOptSymbol *res; + JitOptRef res; REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); res = sym_new_const(ctx, Py_True); stack_pointer[-1] = res; @@ -285,8 +285,8 @@ } case _UNARY_INVERT: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; if (sym_matches_type(value, &PyLong_Type)) { res = sym_new_type(ctx, &PyLong_Type); @@ -299,7 +299,7 @@ } case _GUARD_NOS_INT: { - JitOptSymbol *left; + JitOptRef left; left = stack_pointer[-2]; if (sym_matches_type(left, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -309,7 +309,7 @@ } case _GUARD_TOS_INT: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; if (sym_matches_type(value, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -319,7 +319,7 @@ } case _BINARY_OP_MULTIPLY_INT: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_type(ctx, &PyLong_Type); stack_pointer[-2] = res; stack_pointer += -1; @@ -328,7 +328,7 @@ } case _BINARY_OP_ADD_INT: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_type(ctx, &PyLong_Type); stack_pointer[-2] = res; stack_pointer += -1; @@ -337,7 +337,7 @@ } case _BINARY_OP_SUBTRACT_INT: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_type(ctx, &PyLong_Type); stack_pointer[-2] = res; stack_pointer += -1; @@ -346,7 +346,7 @@ } case _GUARD_NOS_FLOAT: { - JitOptSymbol *left; + JitOptRef left; left = stack_pointer[-2]; if (sym_matches_type(left, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -356,7 +356,7 @@ } case _GUARD_TOS_FLOAT: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; if (sym_matches_type(value, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -366,9 +366,9 @@ } case _BINARY_OP_MULTIPLY_FLOAT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { @@ -390,14 +390,17 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } stack_pointer[-1] = res; break; } case _BINARY_OP_ADD_FLOAT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { @@ -419,14 +422,17 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } stack_pointer[-1] = res; break; } case _BINARY_OP_SUBTRACT_FLOAT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { @@ -448,14 +454,44 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } stack_pointer[-1] = res; break; } + case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: { + JitOptRef res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: { + JitOptRef res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: { + JitOptRef res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _BINARY_OP_ADD_UNICODE: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { @@ -480,11 +516,11 @@ } case _BINARY_OP_INPLACE_ADD_UNICODE: { - JitOptSymbol *right; - JitOptSymbol *left; + JitOptRef right; + JitOptRef left; right = stack_pointer[-1]; left = stack_pointer[-2]; - JitOptSymbol *res; + JitOptRef res; if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); @@ -509,7 +545,7 @@ } case _BINARY_OP_EXTEND: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -518,8 +554,8 @@ } case _BINARY_SLICE: { - JitOptSymbol *container; - JitOptSymbol *res; + JitOptRef container; + JitOptRef res; container = stack_pointer[-3]; PyTypeObject *type = sym_get_type(container); if (type == &PyUnicode_Type || @@ -544,7 +580,7 @@ } case _BINARY_OP_SUBSCR_LIST_INT: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -553,7 +589,7 @@ } case _BINARY_OP_SUBSCR_LIST_SLICE: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -562,7 +598,7 @@ } case _BINARY_OP_SUBSCR_STR_INT: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_type(ctx, &PyUnicode_Type); stack_pointer[-2] = res; stack_pointer += -1; @@ -571,7 +607,7 @@ } case _GUARD_NOS_TUPLE: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -581,7 +617,7 @@ } case _GUARD_TOS_TUPLE: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -591,9 +627,9 @@ } case _BINARY_OP_SUBSCR_TUPLE_INT: { - JitOptSymbol *sub_st; - JitOptSymbol *tuple_st; - JitOptSymbol *res; + JitOptRef sub_st; + JitOptRef tuple_st; + JitOptRef res; sub_st = stack_pointer[-1]; tuple_st = stack_pointer[-2]; assert(sym_matches_type(tuple_st, &PyTuple_Type)); @@ -620,7 +656,7 @@ } case _GUARD_NOS_DICT: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -630,7 +666,7 @@ } case _GUARD_TOS_DICT: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -640,7 +676,7 @@ } case _BINARY_OP_SUBSCR_DICT: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -649,7 +685,7 @@ } case _BINARY_OP_SUBSCR_CHECK_FUNC: { - JitOptSymbol *getitem; + JitOptRef getitem; getitem = sym_new_not_null(ctx); stack_pointer[0] = getitem; stack_pointer += 1; @@ -658,8 +694,8 @@ } case _BINARY_OP_SUBSCR_INIT_CALL: { - JitOptSymbol *new_frame; - new_frame = NULL; + JitOptRef new_frame; + new_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-3] = new_frame; stack_pointer += -2; @@ -704,14 +740,14 @@ } case _CALL_INTRINSIC_1: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _CALL_INTRINSIC_2: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -720,10 +756,10 @@ } case _RETURN_VALUE: { - JitOptSymbol *retval; - JitOptSymbol *res; + JitOptRef retval; + JitOptRef res; retval = stack_pointer[-1]; - JitOptSymbol *temp = retval; + JitOptRef temp = retval; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); ctx->frame->stack_pointer = stack_pointer; @@ -747,14 +783,14 @@ } case _GET_AITER: { - JitOptSymbol *iter; + JitOptRef iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } case _GET_ANEXT: { - JitOptSymbol *awaitable; + JitOptRef awaitable; awaitable = sym_new_not_null(ctx); stack_pointer[0] = awaitable; stack_pointer += 1; @@ -763,7 +799,7 @@ } case _GET_AWAITABLE: { - JitOptSymbol *iter; + JitOptRef iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; @@ -772,15 +808,15 @@ /* _SEND is not a viable micro-op for tier 2 */ case _SEND_GEN_FRAME: { - JitOptSymbol *gen_frame; - gen_frame = NULL; + JitOptRef gen_frame; + gen_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-1] = gen_frame; break; } case _YIELD_VALUE: { - JitOptSymbol *value; + JitOptRef value; value = sym_new_unknown(ctx); stack_pointer[-1] = value; break; @@ -793,7 +829,7 @@ } case _LOAD_COMMON_CONSTANT: { - JitOptSymbol *value; + JitOptRef value; value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; @@ -802,7 +838,7 @@ } case _LOAD_BUILD_CLASS: { - JitOptSymbol *bc; + JitOptRef bc; bc = sym_new_not_null(ctx); stack_pointer[0] = bc; stack_pointer += 1; @@ -821,8 +857,8 @@ } case _UNPACK_SEQUENCE: { - JitOptSymbol **values; - JitOptSymbol **top; + JitOptRef *values; + JitOptRef *top; values = &stack_pointer[-1]; top = &stack_pointer[-1 + oparg]; (void)top; @@ -835,9 +871,9 @@ } case _UNPACK_SEQUENCE_TWO_TUPLE: { - JitOptSymbol *seq; - JitOptSymbol *val1; - JitOptSymbol *val0; + JitOptRef seq; + JitOptRef val1; + JitOptRef val0; seq = stack_pointer[-1]; val0 = sym_tuple_getitem(ctx, seq, 0); val1 = sym_tuple_getitem(ctx, seq, 1); @@ -849,8 +885,8 @@ } case _UNPACK_SEQUENCE_TUPLE: { - JitOptSymbol *seq; - JitOptSymbol **values; + JitOptRef seq; + JitOptRef *values; seq = stack_pointer[-1]; values = &stack_pointer[-1]; for (int i = 0; i < oparg; i++) { @@ -862,7 +898,7 @@ } case _UNPACK_SEQUENCE_LIST: { - JitOptSymbol **values; + JitOptRef *values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_not_null(ctx); @@ -873,8 +909,8 @@ } case _UNPACK_EX: { - JitOptSymbol **values; - JitOptSymbol **top; + JitOptRef *values; + JitOptRef *top; values = &stack_pointer[-1]; top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; (void)top; @@ -910,7 +946,7 @@ } case _LOAD_LOCALS: { - JitOptSymbol *locals; + JitOptRef locals; locals = sym_new_not_null(ctx); stack_pointer[0] = locals; stack_pointer += 1; @@ -921,7 +957,7 @@ /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 */ case _LOAD_NAME: { - JitOptSymbol *v; + JitOptRef v; v = sym_new_not_null(ctx); stack_pointer[0] = v; stack_pointer += 1; @@ -930,7 +966,7 @@ } case _LOAD_GLOBAL: { - JitOptSymbol **res; + JitOptRef *res; res = &stack_pointer[0]; res[0] = sym_new_not_null(ctx); stack_pointer += 1; @@ -939,7 +975,7 @@ } case _PUSH_NULL_CONDITIONAL: { - JitOptSymbol **null; + JitOptRef *null; null = &stack_pointer[0]; if (oparg & 1) { REPLACE_OP(this_instr, _PUSH_NULL, 0, 0); @@ -958,7 +994,7 @@ } case _LOAD_GLOBAL_MODULE: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -967,7 +1003,7 @@ } case _LOAD_GLOBAL_BUILTINS: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -988,14 +1024,14 @@ } case _LOAD_FROM_DICT_OR_DEREF: { - JitOptSymbol *value; + JitOptRef value; value = sym_new_not_null(ctx); stack_pointer[-1] = value; break; } case _LOAD_DEREF: { - JitOptSymbol *value; + JitOptRef value; value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; @@ -1014,7 +1050,7 @@ } case _BUILD_STRING: { - JitOptSymbol *str; + JitOptRef str; str = sym_new_type(ctx, &PyUnicode_Type); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; @@ -1023,7 +1059,7 @@ } case _BUILD_INTERPOLATION: { - JitOptSymbol *interpolation; + JitOptRef interpolation; interpolation = sym_new_not_null(ctx); stack_pointer[-2 - (oparg & 1)] = interpolation; stack_pointer += -1 - (oparg & 1); @@ -1032,7 +1068,7 @@ } case _BUILD_TEMPLATE: { - JitOptSymbol *template; + JitOptRef template; template = sym_new_not_null(ctx); stack_pointer[-2] = template; stack_pointer += -1; @@ -1041,8 +1077,8 @@ } case _BUILD_TUPLE: { - JitOptSymbol **values; - JitOptSymbol *tup; + JitOptRef *values; + JitOptRef tup; values = &stack_pointer[-oparg]; tup = sym_new_tuple(ctx, oparg, values); stack_pointer[-oparg] = tup; @@ -1052,7 +1088,7 @@ } case _BUILD_LIST: { - JitOptSymbol *list; + JitOptRef list; list = sym_new_type(ctx, &PyList_Type); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; @@ -1073,7 +1109,7 @@ } case _BUILD_SET: { - JitOptSymbol *set; + JitOptRef set; set = sym_new_type(ctx, &PySet_Type); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; @@ -1082,7 +1118,7 @@ } case _BUILD_MAP: { - JitOptSymbol *map; + JitOptRef map; map = sym_new_type(ctx, &PyDict_Type); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; @@ -1113,7 +1149,7 @@ } case _LOAD_SUPER_ATTR_ATTR: { - JitOptSymbol *attr_st; + JitOptRef attr_st; attr_st = sym_new_not_null(ctx); stack_pointer[-3] = attr_st; stack_pointer += -2; @@ -1122,8 +1158,8 @@ } case _LOAD_SUPER_ATTR_METHOD: { - JitOptSymbol *attr; - JitOptSymbol *self_or_null; + JitOptRef attr; + JitOptRef self_or_null; attr = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); stack_pointer[-3] = attr; @@ -1134,9 +1170,9 @@ } case _LOAD_ATTR: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol **self_or_null; + JitOptRef owner; + JitOptRef attr; + JitOptRef *self_or_null; owner = stack_pointer[-1]; self_or_null = &stack_pointer[0]; (void)owner; @@ -1151,7 +1187,7 @@ } case _GUARD_TYPE_VERSION: { - JitOptSymbol *owner; + JitOptRef owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)this_instr->operand0; assert(type_version); @@ -1178,7 +1214,7 @@ } case _LOAD_ATTR_INSTANCE_VALUE: { - JitOptSymbol *attr; + JitOptRef attr; uint16_t offset = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); (void)offset; @@ -1187,14 +1223,14 @@ } case _LOAD_ATTR_MODULE: { - JitOptSymbol *owner; - JitOptSymbol *attr; + JitOptRef owner; + JitOptRef attr; owner = stack_pointer[-1]; uint32_t dict_version = (uint32_t)this_instr->operand0; uint16_t index = (uint16_t)this_instr->operand0; (void)dict_version; (void)index; - attr = NULL; + attr = PyJitRef_NULL; if (sym_is_const(ctx, owner)) { PyModuleObject *mod = (PyModuleObject *)sym_get_const(ctx, owner); if (PyModule_CheckExact(mod)) { @@ -1209,7 +1245,7 @@ } } } - if (attr == NULL) { + if (PyJitRef_IsNull(attr)) { attr = sym_new_not_null(ctx); } stack_pointer[-1] = attr; @@ -1217,7 +1253,7 @@ } case _LOAD_ATTR_WITH_HINT: { - JitOptSymbol *attr; + JitOptRef attr; uint16_t hint = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); (void)hint; @@ -1226,7 +1262,7 @@ } case _LOAD_ATTR_SLOT: { - JitOptSymbol *attr; + JitOptRef attr; uint16_t index = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); (void)index; @@ -1235,7 +1271,7 @@ } case _CHECK_ATTR_CLASS: { - JitOptSymbol *owner; + JitOptRef owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)this_instr->operand0; PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version); @@ -1251,8 +1287,8 @@ } case _LOAD_ATTR_CLASS: { - JitOptSymbol *owner; - JitOptSymbol *attr; + JitOptRef owner; + JitOptRef attr; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -1266,10 +1302,10 @@ } case _LOAD_ATTR_PROPERTY_FRAME: { - JitOptSymbol *new_frame; + JitOptRef new_frame; PyObject *fget = (PyObject *)this_instr->operand0; (void)fget; - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-1] = new_frame; break; @@ -1300,7 +1336,7 @@ } case _COMPARE_OP: { - JitOptSymbol *res; + JitOptRef res; if (oparg & 16) { res = sym_new_type(ctx, &PyBool_Type); } @@ -1314,7 +1350,7 @@ } case _COMPARE_OP_FLOAT: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; @@ -1323,9 +1359,9 @@ } case _COMPARE_OP_INT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { @@ -1355,7 +1391,7 @@ } case _COMPARE_OP_STR: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; @@ -1364,7 +1400,7 @@ } case _IS_OP: { - JitOptSymbol *b; + JitOptRef b; b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; @@ -1373,7 +1409,7 @@ } case _CONTAINS_OP: { - JitOptSymbol *b; + JitOptRef b; b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; @@ -1382,7 +1418,7 @@ } case _GUARD_TOS_ANY_SET: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PySet_Type) || sym_matches_type(tos, &PyFrozenSet_Type)) @@ -1393,7 +1429,7 @@ } case _CONTAINS_OP_SET: { - JitOptSymbol *b; + JitOptRef b; b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; @@ -1402,7 +1438,7 @@ } case _CONTAINS_OP_DICT: { - JitOptSymbol *b; + JitOptRef b; b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; @@ -1411,8 +1447,8 @@ } case _CHECK_EG_MATCH: { - JitOptSymbol *rest; - JitOptSymbol *match; + JitOptRef rest; + JitOptRef match; rest = sym_new_not_null(ctx); match = sym_new_not_null(ctx); stack_pointer[-2] = rest; @@ -1421,14 +1457,14 @@ } case _CHECK_EXC_MATCH: { - JitOptSymbol *b; + JitOptRef b; b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } case _IMPORT_NAME: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -1437,7 +1473,7 @@ } case _IMPORT_FROM: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1450,15 +1486,15 @@ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ case _IS_NONE: { - JitOptSymbol *b; + JitOptRef b; b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } case _GET_LEN: { - JitOptSymbol *obj; - JitOptSymbol *len; + JitOptRef obj; + JitOptRef len; obj = stack_pointer[-1]; int tuple_length = sym_tuple_length(obj); if (tuple_length == -1) { @@ -1487,7 +1523,7 @@ } case _MATCH_CLASS: { - JitOptSymbol *attrs; + JitOptRef attrs; attrs = sym_new_not_null(ctx); stack_pointer[-3] = attrs; stack_pointer += -2; @@ -1496,7 +1532,7 @@ } case _MATCH_MAPPING: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1505,7 +1541,7 @@ } case _MATCH_SEQUENCE: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1514,7 +1550,7 @@ } case _MATCH_KEYS: { - JitOptSymbol *values_or_none; + JitOptRef values_or_none; values_or_none = sym_new_not_null(ctx); stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -1523,9 +1559,9 @@ } case _GET_ITER: { - JitOptSymbol *iterable; - JitOptSymbol *iter; - JitOptSymbol *index_or_null; + JitOptRef iterable; + JitOptRef iter; + JitOptRef index_or_null; iterable = stack_pointer[-1]; if (sym_matches_type(iterable, &PyTuple_Type) || sym_matches_type(iterable, &PyList_Type)) { iter = iterable; @@ -1543,7 +1579,7 @@ } case _GET_YIELD_FROM_ITER: { - JitOptSymbol *iter; + JitOptRef iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; @@ -1552,7 +1588,7 @@ /* _FOR_ITER is not a viable micro-op for tier 2 */ case _FOR_ITER_TIER_TWO: { - JitOptSymbol *next; + JitOptRef next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1575,7 +1611,7 @@ /* _ITER_NEXT_LIST is not a viable micro-op for tier 2 */ case _ITER_NEXT_LIST_TIER_TWO: { - JitOptSymbol *next; + JitOptRef next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1584,7 +1620,7 @@ } case _ITER_CHECK_TUPLE: { - JitOptSymbol *iter; + JitOptRef iter; iter = stack_pointer[-2]; if (sym_matches_type(iter, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -1600,7 +1636,7 @@ } case _ITER_NEXT_TUPLE: { - JitOptSymbol *next; + JitOptRef next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1619,7 +1655,7 @@ } case _ITER_NEXT_RANGE: { - JitOptSymbol *next; + JitOptRef next; next = sym_new_type(ctx, &PyLong_Type); stack_pointer[0] = next; stack_pointer += 1; @@ -1628,8 +1664,8 @@ } case _FOR_ITER_GEN_FRAME: { - JitOptSymbol *gen_frame; - gen_frame = NULL; + JitOptRef gen_frame; + gen_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[0] = gen_frame; stack_pointer += 1; @@ -1638,8 +1674,8 @@ } case _INSERT_NULL: { - JitOptSymbol *self; - JitOptSymbol **method_and_self; + JitOptRef self; + JitOptRef *method_and_self; self = stack_pointer[-1]; method_and_self = &stack_pointer[-1]; method_and_self[0] = sym_new_null(ctx); @@ -1650,7 +1686,7 @@ } case _LOAD_SPECIAL: { - JitOptSymbol **method_and_self; + JitOptRef *method_and_self; method_and_self = &stack_pointer[-2]; method_and_self[0] = sym_new_not_null(ctx); method_and_self[1] = sym_new_unknown(ctx); @@ -1658,7 +1694,7 @@ } case _WITH_EXCEPT_START: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1667,8 +1703,8 @@ } case _PUSH_EXC_INFO: { - JitOptSymbol *prev_exc; - JitOptSymbol *new_exc; + JitOptRef prev_exc; + JitOptRef new_exc; prev_exc = sym_new_not_null(ctx); new_exc = sym_new_not_null(ctx); stack_pointer[-1] = prev_exc; @@ -1687,9 +1723,9 @@ } case _LOAD_ATTR_METHOD_WITH_VALUES: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol *self; + JitOptRef owner; + JitOptRef attr; + JitOptRef self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -1707,9 +1743,9 @@ } case _LOAD_ATTR_METHOD_NO_DICT: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol *self; + JitOptRef owner; + JitOptRef attr; + JitOptRef self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -1727,8 +1763,8 @@ } case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { - JitOptSymbol *owner; - JitOptSymbol *attr; + JitOptRef owner; + JitOptRef attr; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -1742,8 +1778,8 @@ } case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { - JitOptSymbol *owner; - JitOptSymbol *attr; + JitOptRef owner; + JitOptRef attr; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -1761,9 +1797,9 @@ } case _LOAD_ATTR_METHOD_LAZY_DICT: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol *self; + JitOptRef owner; + JitOptRef attr; + JitOptRef self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -1781,9 +1817,9 @@ } case _MAYBE_EXPAND_METHOD: { - JitOptSymbol **args; - JitOptSymbol *self_or_null; - JitOptSymbol *callable; + JitOptRef *args; + JitOptRef self_or_null; + JitOptRef callable; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; @@ -1800,7 +1836,7 @@ /* _MONITOR_CALL is not a viable micro-op for tier 2 */ case _PY_FRAME_GENERAL: { - JitOptSymbol *new_frame; + JitOptRef new_frame; PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); co = get_code_with_logging((this_instr + 2)); @@ -1808,7 +1844,7 @@ ctx->done = true; break; } - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1816,7 +1852,7 @@ } case _CHECK_FUNCTION_VERSION: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)this_instr->operand0; if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { @@ -1833,6 +1869,16 @@ } case _CHECK_METHOD_VERSION: { + JitOptRef callable; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = (uint32_t)this_instr->operand0; + if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { + PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); + assert(PyMethod_Check(method)); + REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + this_instr->operand1 = (uintptr_t)method->im_func; + } + sym_set_type(callable, &PyMethod_Type); break; } @@ -1845,7 +1891,7 @@ } case _CALL_NON_PY_GENERAL: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1854,8 +1900,8 @@ } case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { - JitOptSymbol *null; - JitOptSymbol *callable; + JitOptRef null; + JitOptRef callable; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; sym_set_null(null); @@ -1864,8 +1910,8 @@ } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - JitOptSymbol *self_or_null; - JitOptSymbol *callable; + JitOptRef self_or_null; + JitOptRef callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; callable = sym_new_not_null(ctx); @@ -1883,8 +1929,8 @@ } case _CHECK_FUNCTION_EXACT_ARGS: { - JitOptSymbol *self_or_null; - JitOptSymbol *callable; + JitOptRef self_or_null; + JitOptRef callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(sym_matches_type(callable, &PyFunction_Type)); @@ -1911,9 +1957,9 @@ } case _INIT_CALL_PY_EXACT_ARGS: { - JitOptSymbol **args; - JitOptSymbol *self_or_null; - JitOptSymbol *new_frame; + JitOptRef *args; + JitOptRef self_or_null; + JitOptRef new_frame; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; int argcount = oparg; @@ -1924,16 +1970,16 @@ ctx->done = true; break; } - assert(self_or_null != NULL); + assert(!PyJitRef_IsNull(self_or_null)); assert(args != NULL); if (sym_is_not_null(self_or_null)) { args--; argcount++; } if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, args, argcount); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args, argcount)); } else { - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; @@ -1942,12 +1988,12 @@ } case _PUSH_FRAME: { - JitOptSymbol *new_frame; + JitOptRef new_frame; new_frame = stack_pointer[-1]; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); ctx->frame->stack_pointer = stack_pointer; - ctx->frame = (_Py_UOpsAbstractFrame *)new_frame; + ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); ctx->curr_frame_depth++; stack_pointer = ctx->frame->stack_pointer; co = get_code(this_instr); @@ -1974,7 +2020,7 @@ } case _GUARD_NOS_NULL: { - JitOptSymbol *null; + JitOptRef null; null = stack_pointer[-2]; if (sym_is_null(null)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -1984,7 +2030,7 @@ } case _GUARD_NOS_NOT_NULL: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; if (sym_is_not_null(nos)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -1994,7 +2040,7 @@ } case _GUARD_THIRD_NULL: { - JitOptSymbol *null; + JitOptRef null; null = stack_pointer[-3]; if (sym_is_null(null)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -2004,7 +2050,7 @@ } case _GUARD_CALLABLE_TYPE_1: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -2014,8 +2060,8 @@ } case _CALL_TYPE_1: { - JitOptSymbol *arg; - JitOptSymbol *res; + JitOptRef arg; + JitOptRef res; arg = stack_pointer[-1]; PyObject* type = (PyObject *)sym_get_type(arg); if (type) { @@ -2033,7 +2079,7 @@ } case _GUARD_CALLABLE_STR_1: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; if (sym_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -2043,8 +2089,8 @@ } case _CALL_STR_1: { - JitOptSymbol *arg; - JitOptSymbol *res; + JitOptRef arg; + JitOptRef res; arg = stack_pointer[-1]; if (sym_matches_type(arg, &PyUnicode_Type)) { res = arg; @@ -2059,7 +2105,7 @@ } case _GUARD_CALLABLE_TUPLE_1: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; if (sym_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -2069,8 +2115,8 @@ } case _CALL_TUPLE_1: { - JitOptSymbol *arg; - JitOptSymbol *res; + JitOptRef arg; + JitOptRef res; arg = stack_pointer[-1]; if (sym_matches_type(arg, &PyTuple_Type)) { res = arg; @@ -2085,9 +2131,9 @@ } case _CHECK_AND_ALLOCATE_OBJECT: { - JitOptSymbol **args; - JitOptSymbol *self_or_null; - JitOptSymbol *callable; + JitOptRef *args; + JitOptRef self_or_null; + JitOptRef callable; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; @@ -2102,8 +2148,8 @@ } case _CREATE_INIT_FRAME: { - JitOptSymbol *init_frame; - init_frame = NULL; + JitOptRef init_frame; + init_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-2 - oparg] = init_frame; stack_pointer += -1 - oparg; @@ -2118,7 +2164,7 @@ } case _CALL_BUILTIN_CLASS: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2127,7 +2173,7 @@ } case _CALL_BUILTIN_O: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2136,7 +2182,7 @@ } case _CALL_BUILTIN_FAST: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2145,7 +2191,7 @@ } case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2154,7 +2200,7 @@ } case _GUARD_CALLABLE_LEN: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; PyObject *len = _PyInterpreterState_GET()->callable_cache.len; if (sym_get_const(ctx, callable) == len) { @@ -2165,7 +2211,7 @@ } case _CALL_LEN: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_type(ctx, &PyLong_Type); stack_pointer[-3] = res; stack_pointer += -2; @@ -2174,7 +2220,7 @@ } case _GUARD_CALLABLE_ISINSTANCE: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-4]; PyObject *isinstance = _PyInterpreterState_GET()->callable_cache.isinstance; if (sym_get_const(ctx, callable) == isinstance) { @@ -2185,9 +2231,9 @@ } case _CALL_ISINSTANCE: { - JitOptSymbol *cls; - JitOptSymbol *instance; - JitOptSymbol *res; + JitOptRef cls; + JitOptRef instance; + JitOptRef res; cls = stack_pointer[-1]; instance = stack_pointer[-2]; res = sym_new_type(ctx, &PyBool_Type); @@ -2208,7 +2254,7 @@ } case _GUARD_CALLABLE_LIST_APPEND: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; PyObject *list_append = _PyInterpreterState_GET()->callable_cache.list_append; if (sym_get_const(ctx, callable) == list_append) { @@ -2225,7 +2271,7 @@ } case _CALL_METHOD_DESCRIPTOR_O: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2234,7 +2280,7 @@ } case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2243,7 +2289,7 @@ } case _CALL_METHOD_DESCRIPTOR_NOARGS: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2252,7 +2298,7 @@ } case _CALL_METHOD_DESCRIPTOR_FAST: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2269,8 +2315,8 @@ /* _DO_CALL_KW is not a viable micro-op for tier 2 */ case _PY_FRAME_KW: { - JitOptSymbol *new_frame; - new_frame = NULL; + JitOptRef new_frame; + new_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-3 - oparg] = new_frame; stack_pointer += -2 - oparg; @@ -2295,7 +2341,7 @@ } case _CALL_KW_NON_PY: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; @@ -2310,14 +2356,14 @@ /* _DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ case _MAKE_FUNCTION: { - JitOptSymbol *func; + JitOptRef func; func = sym_new_not_null(ctx); stack_pointer[-1] = func; break; } case _SET_FUNCTION_ATTRIBUTE: { - JitOptSymbol *func_out; + JitOptRef func_out; func_out = sym_new_not_null(ctx); stack_pointer[-2] = func_out; stack_pointer += -1; @@ -2326,7 +2372,7 @@ } case _RETURN_GENERATOR: { - JitOptSymbol *res; + JitOptRef res; ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; @@ -2348,7 +2394,7 @@ } case _BUILD_SLICE: { - JitOptSymbol *slice; + JitOptRef slice; slice = sym_new_type(ctx, &PySlice_Type); stack_pointer[-oparg] = slice; stack_pointer += 1 - oparg; @@ -2357,21 +2403,21 @@ } case _CONVERT_VALUE: { - JitOptSymbol *result; + JitOptRef result; result = sym_new_not_null(ctx); stack_pointer[-1] = result; break; } case _FORMAT_SIMPLE: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _FORMAT_WITH_SPEC: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -2380,8 +2426,8 @@ } case _COPY: { - JitOptSymbol *bottom; - JitOptSymbol *top; + JitOptRef bottom; + JitOptRef top; bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = bottom; @@ -2392,9 +2438,9 @@ } case _BINARY_OP: { - JitOptSymbol *rhs; - JitOptSymbol *lhs; - JitOptSymbol *res; + JitOptRef rhs; + JitOptRef lhs; + JitOptRef res; rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; bool lhs_int = sym_matches_type(lhs, &PyLong_Type); @@ -2437,11 +2483,11 @@ } case _SWAP: { - JitOptSymbol *top; - JitOptSymbol *bottom; + JitOptRef top; + JitOptRef bottom; top = stack_pointer[-1]; bottom = stack_pointer[-2 - (oparg-2)]; - JitOptSymbol *temp = bottom; + JitOptRef temp = bottom; bottom = top; top = temp; assert(oparg >= 2); @@ -2469,7 +2515,7 @@ /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ case _GUARD_IS_TRUE_POP: { - JitOptSymbol *flag; + JitOptRef flag; flag = stack_pointer[-1]; if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); @@ -2483,7 +2529,7 @@ } case _GUARD_IS_FALSE_POP: { - JitOptSymbol *flag; + JitOptRef flag; flag = stack_pointer[-1]; if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); @@ -2497,7 +2543,7 @@ } case _GUARD_IS_NONE_POP: { - JitOptSymbol *val; + JitOptRef val; val = stack_pointer[-1]; if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); @@ -2515,7 +2561,7 @@ } case _GUARD_IS_NOT_NONE_POP: { - JitOptSymbol *val; + JitOptRef val; val = stack_pointer[-1]; if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); @@ -2563,9 +2609,9 @@ } case _LOAD_CONST_INLINE: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2573,17 +2619,17 @@ } case _POP_TOP_LOAD_CONST_INLINE: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-1] = value; break; } case _LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2609,15 +2655,15 @@ } case _POP_TOP_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-1] = value; break; } case _POP_TWO_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; value = sym_new_not_null(ctx); stack_pointer[-2] = value; stack_pointer += -1; @@ -2626,9 +2672,9 @@ } case _POP_CALL_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-2] = value; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2636,9 +2682,9 @@ } case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-3] = value; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -2646,9 +2692,9 @@ } case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-4] = value; stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); @@ -2656,8 +2702,8 @@ } case _LOAD_CONST_UNDER_INLINE: { - JitOptSymbol *value; - JitOptSymbol *new; + JitOptRef value; + JitOptRef new; value = sym_new_not_null(ctx); new = sym_new_not_null(ctx); stack_pointer[-1] = value; @@ -2668,8 +2714,8 @@ } case _LOAD_CONST_UNDER_INLINE_BORROW: { - JitOptSymbol *value; - JitOptSymbol *new; + JitOptRef value; + JitOptRef new; value = sym_new_not_null(ctx); new = sym_new_not_null(ctx); stack_pointer[-1] = value; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 25de5d83166..838f25ad288 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -88,6 +88,12 @@ out_of_space(JitOptContext *ctx) return &NO_SPACE_SYMBOL; } +JitOptRef +out_of_space_ref(JitOptContext *ctx) +{ + return PyJitRef_Wrap(out_of_space(ctx)); +} + static JitOptSymbol * sym_new(JitOptContext *ctx) { @@ -98,7 +104,7 @@ sym_new(JitOptContext *ctx) return NULL; } ctx->t_arena.ty_curr_number++; - self->tag = JIT_SYM_UNKNOWN_TAG; + self->tag = JIT_SYM_UNKNOWN_TAG;; return self; } @@ -117,25 +123,28 @@ sym_set_bottom(JitOptContext *ctx, JitOptSymbol *sym) } bool -_Py_uop_sym_is_bottom(JitOptSymbol *sym) +_Py_uop_sym_is_bottom(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); return sym->tag == JIT_SYM_BOTTOM_TAG; } bool -_Py_uop_sym_is_not_null(JitOptSymbol *sym) { +_Py_uop_sym_is_not_null(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); return sym->tag == JIT_SYM_NON_NULL_TAG || sym->tag > JIT_SYM_BOTTOM_TAG; } bool -_Py_uop_sym_is_const(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_is_const(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { return true; } if (sym->tag == JIT_SYM_TRUTHINESS_TAG) { JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_sym_truthiness(ctx, value); + int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_Wrap(value)); if (truthiness < 0) { return false; } @@ -146,21 +155,22 @@ _Py_uop_sym_is_const(JitOptContext *ctx, JitOptSymbol *sym) } bool -_Py_uop_sym_is_null(JitOptSymbol *sym) +_Py_uop_sym_is_null(JitOptRef ref) { - return sym->tag == JIT_SYM_NULL_TAG; + return PyJitRef_Unwrap(ref)->tag == JIT_SYM_NULL_TAG; } PyObject * -_Py_uop_sym_get_const(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_get_const(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { return sym->value.value; } if (sym->tag == JIT_SYM_TRUTHINESS_TAG) { JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_sym_truthiness(ctx, value); + int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_Wrap(value)); if (truthiness < 0) { return NULL; } @@ -172,8 +182,9 @@ _Py_uop_sym_get_const(JitOptContext *ctx, JitOptSymbol *sym) } void -_Py_uop_sym_set_type(JitOptContext *ctx, JitOptSymbol *sym, PyTypeObject *typ) +_Py_uop_sym_set_type(JitOptContext *ctx, JitOptRef ref, PyTypeObject *typ) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -222,11 +233,12 @@ _Py_uop_sym_set_type(JitOptContext *ctx, JitOptSymbol *sym, PyTypeObject *typ) } bool -_Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptSymbol *sym, unsigned int version) +_Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptRef ref, unsigned int version) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); PyTypeObject *type = _PyType_LookupByVersion(version); if (type) { - _Py_uop_sym_set_type(ctx, sym, type); + _Py_uop_sym_set_type(ctx, ref, type); } JitSymType tag = sym->tag; switch(tag) { @@ -279,8 +291,9 @@ _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptSymbol *sym, unsigned int } void -_Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val) +_Py_uop_sym_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -301,10 +314,10 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val return; case JIT_SYM_TUPLE_TAG: if (PyTuple_CheckExact(const_val)) { - Py_ssize_t len = _Py_uop_sym_tuple_length(sym); + Py_ssize_t len = _Py_uop_sym_tuple_length(ref); if (len == PyTuple_GET_SIZE(const_val)) { for (Py_ssize_t i = 0; i < len; i++) { - JitOptSymbol *sym_item = _Py_uop_sym_tuple_getitem(ctx, sym, i); + JitOptRef sym_item = _Py_uop_sym_tuple_getitem(ctx, ref, i); PyObject *item = PyTuple_GET_ITEM(const_val, i); _Py_uop_sym_set_const(ctx, sym_item, item); } @@ -329,13 +342,14 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val return; case JIT_SYM_TRUTHINESS_TAG: if (!PyBool_Check(const_val) || - (_Py_uop_sym_is_const(ctx, sym) && - _Py_uop_sym_get_const(ctx, sym) != const_val)) + (_Py_uop_sym_is_const(ctx, ref) && + _Py_uop_sym_get_const(ctx, ref) != const_val)) { sym_set_bottom(ctx, sym); return; } - JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; + JitOptRef value = PyJitRef_Wrap( + allocation_base(ctx) + sym->truthiness.value); PyTypeObject *type = _Py_uop_sym_get_type(value); if (const_val == (sym->truthiness.invert ? Py_False : Py_True)) { // value is truthy. This is only useful for bool: @@ -360,8 +374,9 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val } void -_Py_uop_sym_set_null(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_set_null(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_UNKNOWN_TAG) { sym->tag = JIT_SYM_NULL_TAG; } @@ -371,8 +386,9 @@ _Py_uop_sym_set_null(JitOptContext *ctx, JitOptSymbol *sym) } void -_Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_UNKNOWN_TAG) { sym->tag = JIT_SYM_NON_NULL_TAG; } @@ -381,66 +397,69 @@ _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym) } } - -JitOptSymbol * +JitOptRef _Py_uop_sym_new_unknown(JitOptContext *ctx) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - return res; + return PyJitRef_Wrap(res); } -JitOptSymbol * +JitOptRef _Py_uop_sym_new_not_null(JitOptContext *ctx) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } res->tag = JIT_SYM_NON_NULL_TAG; - return res; + return PyJitRef_Wrap(res); } -JitOptSymbol * +JitOptRef _Py_uop_sym_new_type(JitOptContext *ctx, PyTypeObject *typ) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - _Py_uop_sym_set_type(ctx, res, typ); - return res; + JitOptRef ref = PyJitRef_Wrap(res); + _Py_uop_sym_set_type(ctx, ref, typ); + return ref; } // Adds a new reference to const_val, owned by the symbol. -JitOptSymbol * +JitOptRef _Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val) { assert(const_val != NULL); JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - _Py_uop_sym_set_const(ctx, res, const_val); - return res; + JitOptRef ref = PyJitRef_Wrap(res); + _Py_uop_sym_set_const(ctx, ref, const_val); + return ref; } -JitOptSymbol * +JitOptRef _Py_uop_sym_new_null(JitOptContext *ctx) { JitOptSymbol *null_sym = sym_new(ctx); if (null_sym == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - _Py_uop_sym_set_null(ctx, null_sym); - return null_sym; + JitOptRef ref = PyJitRef_Wrap(null_sym); + _Py_uop_sym_set_null(ctx, ref); + return ref; } PyTypeObject * -_Py_uop_sym_get_type(JitOptSymbol *sym) +_Py_uop_sym_get_type(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -463,8 +482,9 @@ _Py_uop_sym_get_type(JitOptSymbol *sym) } unsigned int -_Py_uop_sym_get_type_version(JitOptSymbol *sym) +_Py_uop_sym_get_type_version(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -487,27 +507,28 @@ _Py_uop_sym_get_type_version(JitOptSymbol *sym) } bool -_Py_uop_sym_has_type(JitOptSymbol *sym) +_Py_uop_sym_has_type(JitOptRef sym) { return _Py_uop_sym_get_type(sym) != NULL; } bool -_Py_uop_sym_matches_type(JitOptSymbol *sym, PyTypeObject *typ) +_Py_uop_sym_matches_type(JitOptRef sym, PyTypeObject *typ) { assert(typ != NULL && PyType_Check(typ)); return _Py_uop_sym_get_type(sym) == typ; } bool -_Py_uop_sym_matches_type_version(JitOptSymbol *sym, unsigned int version) +_Py_uop_sym_matches_type_version(JitOptRef sym, unsigned int version) { return _Py_uop_sym_get_type_version(sym) == version; } int -_Py_uop_sym_truthiness(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_truthiness(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); switch(sym->tag) { case JIT_SYM_NULL_TAG: case JIT_SYM_TYPE_VERSION_TAG: @@ -527,7 +548,8 @@ _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptSymbol *sym) case JIT_SYM_TRUTHINESS_TAG: ; JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_sym_truthiness(ctx, value); + int truthiness = _Py_uop_sym_truthiness(ctx, + PyJitRef_Wrap(value)); if (truthiness < 0) { return truthiness; } @@ -553,12 +575,12 @@ _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptSymbol *sym) return -1; } -JitOptSymbol * -_Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptSymbol **args) +JitOptRef +_Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptRef *args) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } if (size > MAX_SYMBOLIC_TUPLE_SIZE) { res->tag = JIT_SYM_KNOWN_CLASS_TAG; @@ -568,15 +590,16 @@ _Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptSymbol **args) res->tag = JIT_SYM_TUPLE_TAG; res->tuple.length = size; for (int i = 0; i < size; i++) { - res->tuple.items[i] = (uint16_t)(args[i] - allocation_base(ctx)); + res->tuple.items[i] = (uint16_t)(PyJitRef_Unwrap(args[i]) - allocation_base(ctx)); } } - return res; + return PyJitRef_Wrap(res); } -JitOptSymbol * -_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptSymbol *sym, int item) +JitOptRef +_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptRef ref, int item) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); assert(item >= 0); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { PyObject *tuple = sym->value.value; @@ -585,14 +608,15 @@ _Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptSymbol *sym, int item) } } else if (sym->tag == JIT_SYM_TUPLE_TAG && item < sym->tuple.length) { - return allocation_base(ctx) + sym->tuple.items[item]; + return PyJitRef_Wrap(allocation_base(ctx) + sym->tuple.items[item]); } return _Py_uop_sym_new_not_null(ctx); } int -_Py_uop_sym_tuple_length(JitOptSymbol *sym) +_Py_uop_sym_tuple_length(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { PyObject *tuple = sym->value.value; if (PyTuple_CheckExact(tuple)) { @@ -607,7 +631,7 @@ _Py_uop_sym_tuple_length(JitOptSymbol *sym) // Return true if known to be immortal. bool -_Py_uop_sym_is_immortal(JitOptSymbol *sym) +_Py_uop_symbol_is_immortal(JitOptSymbol *sym) { if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { return _Py_IsImmortal(sym->value.value); @@ -621,19 +645,27 @@ _Py_uop_sym_is_immortal(JitOptSymbol *sym) return false; } -JitOptSymbol * -_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy) +bool +_Py_uop_sym_is_immortal(JitOptRef ref) +{ + JitOptSymbol *sym = PyJitRef_Unwrap(ref); + return _Py_uop_symbol_is_immortal(sym); +} + +JitOptRef +_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptRef ref, bool truthy) { + JitOptSymbol *value = PyJitRef_Unwrap(ref); // It's clearer to invert this in the signature: bool invert = !truthy; if (value->tag == JIT_SYM_TRUTHINESS_TAG && value->truthiness.invert == invert) { - return value; + return ref; } JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - int truthiness = _Py_uop_sym_truthiness(ctx, value); + int truthiness = _Py_uop_sym_truthiness(ctx, ref); if (truthiness < 0) { res->tag = JIT_SYM_TRUTHINESS_TAG; res->truthiness.invert = invert; @@ -642,7 +674,7 @@ _Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy) else { make_const(res, (truthiness ^ invert) ? Py_True : Py_False); } - return res; + return PyJitRef_Wrap(res); } // 0 on success, -1 on error. @@ -651,7 +683,7 @@ _Py_uop_frame_new( JitOptContext *ctx, PyCodeObject *co, int curr_stackentries, - JitOptSymbol **args, + JitOptRef *args, int arg_len) { assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); @@ -676,14 +708,14 @@ _Py_uop_frame_new( } for (int i = arg_len; i < co->co_nlocalsplus; i++) { - JitOptSymbol *local = _Py_uop_sym_new_unknown(ctx); + JitOptRef local = _Py_uop_sym_new_unknown(ctx); frame->locals[i] = local; } // Initialize the stack as well for (int i = 0; i < curr_stackentries; i++) { - JitOptSymbol *stackvar = _Py_uop_sym_new_unknown(ctx); + JitOptRef stackvar = _Py_uop_sym_new_unknown(ctx); frame->stack[i] = stackvar; } @@ -709,12 +741,12 @@ _Py_uop_abstractcontext_fini(JitOptContext *ctx) void _Py_uop_abstractcontext_init(JitOptContext *ctx) { - static_assert(sizeof(JitOptSymbol) <= 2 * sizeof(uint64_t), "JitOptSymbol has grown"); + static_assert(sizeof(JitOptSymbol) <= 3 * sizeof(uint64_t), "JitOptSymbol has grown"); ctx->limit = ctx->locals_and_stack + MAX_ABSTRACT_INTERP_SIZE; ctx->n_consumed = ctx->locals_and_stack; #ifdef Py_DEBUG // Aids debugging a little. There should never be NULL in the abstract interpreter. for (int i = 0 ; i < MAX_ABSTRACT_INTERP_SIZE; i++) { - ctx->locals_and_stack[i] = NULL; + ctx->locals_and_stack[i] = PyJitRef_NULL; } #endif @@ -767,44 +799,44 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) PyObject *tuple = NULL; // Use a single 'sym' variable so copy-pasting tests is easier. - JitOptSymbol *sym = _Py_uop_sym_new_unknown(ctx); - if (sym == NULL) { + JitOptRef ref = _Py_uop_sym_new_unknown(ctx); + if (PyJitRef_IsNull(ref)) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "top is NULL"); - TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "top is not NULL"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "top matches a type"); - TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, sym), "top is a constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "top as constant is not NULL"); - TEST_PREDICATE(!_Py_uop_sym_is_bottom(sym), "top is bottom"); + TEST_PREDICATE(!_Py_uop_sym_is_null(ref), "top is NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_not_null(ref), "top is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(ref, &PyLong_Type), "top matches a type"); + TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, ref), "top is a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == NULL, "top as constant is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_bottom(ref), "top is bottom"); - sym = make_bottom(ctx); - if (sym == NULL) { + ref = PyJitRef_Wrap(make_bottom(ctx)); + if (PyJitRef_IsNull(ref)) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "bottom is NULL is not false"); - TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "bottom is not NULL is not false"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "bottom matches a type"); - TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, sym), "bottom is a constant is not false"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "bottom as constant is not NULL"); - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "bottom isn't bottom"); + TEST_PREDICATE(!_Py_uop_sym_is_null(ref), "bottom is NULL is not false"); + TEST_PREDICATE(!_Py_uop_sym_is_not_null(ref), "bottom is not NULL is not false"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(ref, &PyLong_Type), "bottom matches a type"); + TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, ref), "bottom is a constant is not false"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == NULL, "bottom as constant is not NULL"); + TEST_PREDICATE(_Py_uop_sym_is_bottom(ref), "bottom isn't bottom"); - sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { + ref = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (PyJitRef_IsNull(ref)) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "int is NULL"); - TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "int isn't not NULL"); - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "int isn't int"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "int matches float"); - TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, sym), "int is a constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "int as constant is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_null(ref), "int is NULL"); + TEST_PREDICATE(_Py_uop_sym_is_not_null(ref), "int isn't not NULL"); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyLong_Type), "int isn't int"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(ref, &PyFloat_Type), "int matches float"); + TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, ref), "int is a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == NULL, "int as constant is not NULL"); - _Py_uop_sym_set_type(ctx, sym, &PyLong_Type); // Should be a no-op - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(int and int) isn't int"); + _Py_uop_sym_set_type(ctx, ref, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyLong_Type), "(int and int) isn't int"); - _Py_uop_sym_set_type(ctx, sym, &PyFloat_Type); // Should make it bottom - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(int and float) isn't bottom"); + _Py_uop_sym_set_type(ctx, ref, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(ref), "(int and float) isn't bottom"); val_42 = PyLong_FromLong(42); assert(val_42 != NULL); @@ -814,84 +846,84 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) assert(val_43 != NULL); assert(_Py_IsImmortal(val_43)); - sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { + ref = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (PyJitRef_IsNull(ref)) { goto fail; } - _Py_uop_sym_set_const(ctx, sym, val_42); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 1, "bool(42) is not True"); - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "42 is NULL"); - TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "42 isn't not NULL"); - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "42 isn't an int"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "42 matches float"); - TEST_PREDICATE(_Py_uop_sym_is_const(ctx, sym), "42 is not a constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) != NULL, "42 as constant is NULL"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == val_42, "42 as constant isn't 42"); - TEST_PREDICATE(_Py_uop_sym_is_immortal(sym), "42 is not immortal"); - - _Py_uop_sym_set_type(ctx, sym, &PyLong_Type); // Should be a no-op - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(42 and 42) isn't an int"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == val_42, "(42 and 42) as constant isn't 42"); - - _Py_uop_sym_set_type(ctx, sym, &PyFloat_Type); // Should make it bottom - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and float) isn't bottom"); - - sym = _Py_uop_sym_new_type(ctx, &PyBool_Type); - TEST_PREDICATE(_Py_uop_sym_is_immortal(sym), "a bool is not immortal"); - - sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { + _Py_uop_sym_set_const(ctx, ref, val_42); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 1, "bool(42) is not True"); + TEST_PREDICATE(!_Py_uop_sym_is_null(ref), "42 is NULL"); + TEST_PREDICATE(_Py_uop_sym_is_not_null(ref), "42 isn't not NULL"); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyLong_Type), "42 isn't an int"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(ref, &PyFloat_Type), "42 matches float"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, ref), "42 is not a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) != NULL, "42 as constant is NULL"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == val_42, "42 as constant isn't 42"); + TEST_PREDICATE(_Py_uop_sym_is_immortal(ref), "42 is not immortal"); + + _Py_uop_sym_set_type(ctx, ref, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyLong_Type), "(42 and 42) isn't an int"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == val_42, "(42 and 42) as constant isn't 42"); + + _Py_uop_sym_set_type(ctx, ref, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(ref), "(42 and float) isn't bottom"); + + ref = _Py_uop_sym_new_type(ctx, &PyBool_Type); + TEST_PREDICATE(_Py_uop_sym_is_immortal(ref), "a bool is not immortal"); + + ref = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (PyJitRef_IsNull(ref)) { goto fail; } - _Py_uop_sym_set_const(ctx, sym, val_42); - _Py_uop_sym_set_const(ctx, sym, val_43); // Should make it bottom - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and 43) isn't bottom"); + _Py_uop_sym_set_const(ctx, ref, val_42); + _Py_uop_sym_set_const(ctx, ref, val_43); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(ref), "(42 and 43) isn't bottom"); - sym = _Py_uop_sym_new_const(ctx, Py_None); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(None) is not False"); - sym = _Py_uop_sym_new_const(ctx, Py_False); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(False) is not False"); - sym = _Py_uop_sym_new_const(ctx, PyLong_FromLong(0)); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(0) is not False"); + ref = _Py_uop_sym_new_const(ctx, Py_None); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 0, "bool(None) is not False"); + ref = _Py_uop_sym_new_const(ctx, Py_False); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 0, "bool(False) is not False"); + ref = _Py_uop_sym_new_const(ctx, PyLong_FromLong(0)); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 0, "bool(0) is not False"); - JitOptSymbol *i1 = _Py_uop_sym_new_type(ctx, &PyFloat_Type); - JitOptSymbol *i2 = _Py_uop_sym_new_const(ctx, val_43); - JitOptSymbol *array[2] = { i1, i2 }; - sym = _Py_uop_sym_new_tuple(ctx, 2, array); + JitOptRef i1 = _Py_uop_sym_new_type(ctx, &PyFloat_Type); + JitOptRef i2 = _Py_uop_sym_new_const(ctx, val_43); + JitOptRef array[2] = { i1, i2 }; + ref = _Py_uop_sym_new_tuple(ctx, 2, array); TEST_PREDICATE( - _Py_uop_sym_matches_type(_Py_uop_sym_tuple_getitem(ctx, sym, 0), &PyFloat_Type), + _Py_uop_sym_matches_type(_Py_uop_sym_tuple_getitem(ctx, ref, 0), &PyFloat_Type), "tuple item does not match value used to create tuple" ); TEST_PREDICATE( - _Py_uop_sym_get_const(ctx, _Py_uop_sym_tuple_getitem(ctx, sym, 1)) == val_43, + _Py_uop_sym_get_const(ctx, _Py_uop_sym_tuple_getitem(ctx, ref, 1)) == val_43, "tuple item does not match value used to create tuple" ); PyObject *pair[2] = { val_42, val_43 }; tuple = _PyTuple_FromArray(pair, 2); - sym = _Py_uop_sym_new_const(ctx, tuple); + ref = _Py_uop_sym_new_const(ctx, tuple); TEST_PREDICATE( - _Py_uop_sym_get_const(ctx, _Py_uop_sym_tuple_getitem(ctx, sym, 1)) == val_43, + _Py_uop_sym_get_const(ctx, _Py_uop_sym_tuple_getitem(ctx, ref, 1)) == val_43, "tuple item does not match value used to create tuple" ); - sym = _Py_uop_sym_new_type(ctx, &PyTuple_Type); + ref = _Py_uop_sym_new_type(ctx, &PyTuple_Type); TEST_PREDICATE( - _Py_uop_sym_is_not_null(_Py_uop_sym_tuple_getitem(ctx, sym, 42)), + _Py_uop_sym_is_not_null(_Py_uop_sym_tuple_getitem(ctx, ref, 42)), "Unknown tuple item is not narrowed to non-NULL" ); - JitOptSymbol *value = _Py_uop_sym_new_type(ctx, &PyBool_Type); - sym = _Py_uop_sym_new_truthiness(ctx, value, false); - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyBool_Type), "truthiness is not boolean"); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == -1, "truthiness is not unknown"); - TEST_PREDICATE(_Py_uop_sym_is_const(ctx, sym) == false, "truthiness is constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "truthiness is not NULL"); + JitOptRef value = _Py_uop_sym_new_type(ctx, &PyBool_Type); + ref = _Py_uop_sym_new_truthiness(ctx, value, false); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyBool_Type), "truthiness is not boolean"); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == -1, "truthiness is not unknown"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, ref) == false, "truthiness is constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == NULL, "truthiness is not NULL"); TEST_PREDICATE(_Py_uop_sym_is_const(ctx, value) == false, "value is constant"); TEST_PREDICATE(_Py_uop_sym_get_const(ctx, value) == NULL, "value is not NULL"); - _Py_uop_sym_set_const(ctx, sym, Py_False); - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyBool_Type), "truthiness is not boolean"); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "truthiness is not True"); - TEST_PREDICATE(_Py_uop_sym_is_const(ctx, sym) == true, "truthiness is not constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == Py_False, "truthiness is not False"); + _Py_uop_sym_set_const(ctx, ref, Py_False); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyBool_Type), "truthiness is not boolean"); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 0, "truthiness is not True"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, ref) == true, "truthiness is not constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == Py_False, "truthiness is not False"); TEST_PREDICATE(_Py_uop_sym_is_const(ctx, value) == true, "value is not constant"); TEST_PREDICATE(_Py_uop_sym_get_const(ctx, value) == Py_True, "value is not True"); _Py_uop_abstractcontext_fini(ctx); diff --git a/Python/pystate.c b/Python/pystate.c index 0544b15aad1..0d4c26f92ce 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1142,6 +1142,7 @@ _Py_CheckMainModule(PyObject *module) PyObject *msg = PyUnicode_FromString("invalid __main__ module"); if (msg != NULL) { (void)PyErr_SetImportError(msg, &_Py_ID(__main__), NULL); + Py_DECREF(msg); } return -1; } diff --git a/Python/specialize.c b/Python/specialize.c index 92f79d39d55..fe8d04cf344 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -118,6 +118,7 @@ _Py_GetSpecializationStats(void) { err += add_stat_dict(stats, LOAD_GLOBAL, "load_global"); err += add_stat_dict(stats, STORE_SUBSCR, "store_subscr"); err += add_stat_dict(stats, STORE_ATTR, "store_attr"); + err += add_stat_dict(stats, JUMP_BACKWARD, "jump_backward"); err += add_stat_dict(stats, CALL, "call"); err += add_stat_dict(stats, CALL_KW, "call_kw"); err += add_stat_dict(stats, BINARY_OP, "binary_op"); |