Skip to content

Commit

Permalink
bpo-46841: Quicken code in-place (GH-31888)
Browse files Browse the repository at this point in the history 
* Moves the bytecode to the end of the corresponding PyCodeObject, and quickens it in-place.

* Removes the almost-always-unused co_varnames, co_freevars, and co_cellvars member caches

* _PyOpcode_Deopt is a new mapping from all opcodes to their un-quickened forms.

* _PyOpcode_InlineCacheEntries is renamed to _PyOpcode_Caches

* _Py_IncrementCountAndMaybeQuicken is renamed to _PyCode_Warmup

* _Py_Quicken is renamed to _PyCode_Quicken

* _co_quickened is renamed to _co_code_adaptive (and is now a read-only memoryview).

* Do not emit unused nonzero opargs anymore in the compiler.
  • Loading branch information
@brandtbucher
brandtbucher authored Mar 21, 2022
1 parent 08eb754 commit 2bde682
Show file tree
Hide file tree
Showing 18 changed files with 831 additions and 687 deletions.
157 changes: 74 additions & 83 deletions Include/cpython/code.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,91 +26,80 @@ typedef uint16_t _Py_CODEUNIT;
// Use "unsigned char" instead of "uint8_t" here to avoid illegal aliasing:
#define _Py_SET_OPCODE(word, opcode) (((unsigned char *)&(word))[0] = (opcode))

// To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are
// defined in this macro:
#define _PyCode_DEF(SIZE) { \
PyObject_VAR_HEAD \
\
/* Note only the following fields are used in hash and/or comparisons \
* \
* - co_name \
* - co_argcount \
* - co_posonlyargcount \
* - co_kwonlyargcount \
* - co_nlocals \
* - co_stacksize \
* - co_flags \
* - co_firstlineno \
* - co_consts \
* - co_names \
* - co_localsplusnames \
* This is done to preserve the name and line number for tracebacks \
* and debuggers; otherwise, constant de-duplication would collapse \
* identical functions/lambdas defined on different lines. \
*/ \
\
/* These fields are set with provided values on new code objects. */ \
\
/* The hottest fields (in the eval loop) are grouped here at the top. */ \
PyObject *co_consts; /* list (constants used) */ \
PyObject *co_names; /* list of strings (names used) */ \
PyObject *co_exceptiontable; /* Byte string encoding exception handling \
table */ \
int co_flags; /* CO_..., see below */ \
int co_warmup; /* Warmup counter for quickening */ \
\
/* The rest are not so impactful on performance. */ \
int co_argcount; /* #arguments, except *args */ \
int co_posonlyargcount; /* #positional only arguments */ \
int co_kwonlyargcount; /* #keyword only arguments */ \
int co_stacksize; /* #entries needed for evaluation stack */ \
int co_firstlineno; /* first source line number */ \
\
/* redundant values (derived from co_localsplusnames and \
co_localspluskinds) */ \
int co_nlocalsplus; /* number of local + cell + free variables \
*/ \
int co_nlocals; /* number of local variables */ \
int co_nplaincellvars; /* number of non-arg cell variables */ \
int co_ncellvars; /* total number of cell variables */ \
int co_nfreevars; /* number of free variables */ \
\
PyObject *co_localsplusnames; /* tuple mapping offsets to names */ \
PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte \
per variable) */ \
PyObject *co_filename; /* unicode (where it was loaded from) */ \
PyObject *co_name; /* unicode (name, for reference) */ \
PyObject *co_qualname; /* unicode (qualname, for reference) */ \
PyObject *co_linetable; /* bytes (encoding addr<->lineno mapping) \
See Objects/lnotab_notes.txt for details. \
*/ \
PyObject *co_endlinetable; /* bytes object that holds end lineno for \
instructions separated across different \
lines */ \
PyObject *co_columntable; /* bytes object that holds start/end column \
offset each instruction */ \
\
PyObject *co_weakreflist; /* to support weakrefs to code objects */ \
/* Scratch space for extra data relating to the code object. \
Type is a void* to keep the format private in codeobject.c to force \
people to go through the proper APIs. */ \
void *co_extra; \
char co_code_adaptive[(SIZE)]; \
}

/* Bytecode object */
struct PyCodeObject {
PyObject_HEAD

/* Note only the following fields are used in hash and/or comparisons
*
* - co_name
* - co_argcount
* - co_posonlyargcount
* - co_kwonlyargcount
* - co_nlocals
* - co_stacksize
* - co_flags
* - co_firstlineno
* - co_code
* - co_consts
* - co_names
* - co_varnames
* - co_freevars
* - co_cellvars
*
* This is done to preserve the name and line number for tracebacks
* and debuggers; otherwise, constant de-duplication would collapse
* identical functions/lambdas defined on different lines.
*/

/* These fields are set with provided values on new code objects. */

// The hottest fields (in the eval loop) are grouped here at the top.
PyObject *co_consts; /* list (constants used) */
PyObject *co_names; /* list of strings (names used) */
_Py_CODEUNIT *co_firstinstr; /* Pointer to first instruction, used for quickening.
Unlike the other "hot" fields, this one is
actually derived from co_code. */
PyObject *co_exceptiontable; /* Byte string encoding exception handling table */
int co_flags; /* CO_..., see below */
int co_warmup; /* Warmup counter for quickening */

// The rest are not so impactful on performance.
int co_argcount; /* #arguments, except *args */
int co_posonlyargcount; /* #positional only arguments */
int co_kwonlyargcount; /* #keyword only arguments */
int co_stacksize; /* #entries needed for evaluation stack */
int co_firstlineno; /* first source line number */
PyObject *co_code; /* instruction opcodes */
PyObject *co_localsplusnames; /* tuple mapping offsets to names */
PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte per variable) */
PyObject *co_filename; /* unicode (where it was loaded from) */
PyObject *co_name; /* unicode (name, for reference) */
PyObject *co_qualname; /* unicode (qualname, for reference) */
PyObject *co_linetable; /* bytes (encoding addr<->lineno mapping) See
Objects/lnotab_notes.txt for details. */
PyObject *co_endlinetable; /* bytes object that holds end lineno for
instructions separated across different
lines */
PyObject *co_columntable; /* bytes object that holds start/end column
offset each instruction */

/* These fields are set with computed values on new code objects. */

// redundant values (derived from co_localsplusnames and co_localspluskinds)
int co_nlocalsplus; /* number of local + cell + free variables */
int co_nlocals; /* number of local variables */
int co_nplaincellvars; /* number of non-arg cell variables */
int co_ncellvars; /* total number of cell variables */
int co_nfreevars; /* number of free variables */
// lazily-computed values
PyObject *co_varnames; /* tuple of strings (local variable names) */
PyObject *co_cellvars; /* tuple of strings (cell variable names) */
PyObject *co_freevars; /* tuple of strings (free variable names) */

/* The remaining fields are zeroed out on new code objects. */

PyObject *co_weakreflist; /* to support weakrefs to code objects */
/* Scratch space for extra data relating to the code object.
Type is a void* to keep the format private in codeobject.c to force
people to go through the proper APIs. */
void *co_extra;
/* Quickened instructions and cache, or NULL
This should be treated as opaque by all code except the specializer and
interpreter. */
_Py_CODEUNIT *co_quickened;

};
struct PyCodeObject _PyCode_DEF(1);

/* Masks for co_flags above */
#define CO_OPTIMIZED 0x0001
Expand Down Expand Up @@ -151,6 +140,8 @@ PyAPI_DATA(PyTypeObject) PyCode_Type;

#define PyCode_Check(op) Py_IS_TYPE(op, &PyCode_Type)
#define PyCode_GetNumFree(op) ((op)->co_nfreevars)
#define _PyCode_CODE(CO) ((_Py_CODEUNIT *)(CO)->co_code_adaptive)
#define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT))

/* Public interface */
PyAPI_FUNC(PyCodeObject *) PyCode_New(
Expand Down
17 changes: 5 additions & 12 deletions Include/internal/pycore_code.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,30 +92,22 @@ typedef struct {

#define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache)

/* Maximum size of code to quicken, in code units. */
#define MAX_SIZE_TO_QUICKEN 10000

#define QUICKENING_WARMUP_DELAY 8

/* We want to compare to zero for efficiency, so we offset values accordingly */
#define QUICKENING_INITIAL_WARMUP_VALUE (-QUICKENING_WARMUP_DELAY)
#define QUICKENING_WARMUP_COLDEST 1

int _Py_Quicken(PyCodeObject *code);
void _PyCode_Quicken(PyCodeObject *code);

/* Returns 1 if quickening occurs.
* -1 if an error occurs
* 0 otherwise */
static inline int
_Py_IncrementCountAndMaybeQuicken(PyCodeObject *code)
static inline void
_PyCode_Warmup(PyCodeObject *code)
{
if (code->co_warmup != 0) {
code->co_warmup++;
if (code->co_warmup == 0) {
return _Py_Quicken(code) ? -1 : 1;
_PyCode_Quicken(code);
}
}
return 0;
}

extern Py_ssize_t _Py_QuickenedCount;
Expand Down Expand Up @@ -225,6 +217,7 @@ PyAPI_FUNC(PyCodeObject *) _PyCode_New(struct _PyCodeConstructor *);
extern PyObject* _PyCode_GetVarnames(PyCodeObject *);
extern PyObject* _PyCode_GetCellvars(PyCodeObject *);
extern PyObject* _PyCode_GetFreevars(PyCodeObject *);
extern PyObject* _PyCode_GetCode(PyCodeObject *);

/* Return the ending source code line number from a bytecode index. */
extern int _PyCode_Addr2EndLine(PyCodeObject *, int);
Expand Down
Loading

0 comments on commit 2bde682

Please sign in to comment.