python3.12/internal/pycore_code.h

0001 #ifndef Py_INTERNAL_CODE_H
0002 #define Py_INTERNAL_CODE_H
0003 #ifdef __cplusplus
0004 extern "C" {
0005 #endif
0006
0007 #define CODE_MAX_WATCHERS 8
0008
0009 /* PEP 659
0010  * Specialization and quickening structs and helper functions
0011  */
0012
0013
0014 // Inline caches. If you change the number of cache entries for an instruction,
0015 // you must *also* update the number of cache entries in Lib/opcode.py and bump
0016 // the magic number in Lib/importlib/_bootstrap_external.py!
0017
0018 #define CACHE_ENTRIES(cache) (sizeof(cache)/sizeof(_Py_CODEUNIT))
0019
0020 typedef struct {
0021     uint16_t counter;
0022     uint16_t index;
0023     uint16_t module_keys_version;
0024     uint16_t builtin_keys_version;
0025 } _PyLoadGlobalCache;
0026
0027 #define INLINE_CACHE_ENTRIES_LOAD_GLOBAL CACHE_ENTRIES(_PyLoadGlobalCache)
0028
0029 typedef struct {
0030     uint16_t counter;
0031 } _PyBinaryOpCache;
0032
0033 #define INLINE_CACHE_ENTRIES_BINARY_OP CACHE_ENTRIES(_PyBinaryOpCache)
0034
0035 typedef struct {
0036     uint16_t counter;
0037 } _PyUnpackSequenceCache;
0038
0039 #define INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE \
0040     CACHE_ENTRIES(_PyUnpackSequenceCache)
0041
0042 typedef struct {
0043     uint16_t counter;
0044 } _PyCompareOpCache;
0045
0046 #define INLINE_CACHE_ENTRIES_COMPARE_OP CACHE_ENTRIES(_PyCompareOpCache)
0047
0048 typedef struct {
0049     uint16_t counter;
0050 } _PyBinarySubscrCache;
0051
0052 #define INLINE_CACHE_ENTRIES_BINARY_SUBSCR CACHE_ENTRIES(_PyBinarySubscrCache)
0053
0054 typedef struct {
0055     uint16_t counter;
0056 } _PySuperAttrCache;
0057
0058 #define INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR CACHE_ENTRIES(_PySuperAttrCache)
0059
0060 typedef struct {
0061     uint16_t counter;
0062     uint16_t version[2];
0063     uint16_t index;
0064 } _PyAttrCache;
0065
0066 typedef struct {
0067     uint16_t counter;
0068     uint16_t type_version[2];
0069     uint16_t keys_version[2];
0070     uint16_t descr[4];
0071 } _PyLoadMethodCache;
0072
0073
0074 // MUST be the max(_PyAttrCache, _PyLoadMethodCache)
0075 #define INLINE_CACHE_ENTRIES_LOAD_ATTR CACHE_ENTRIES(_PyLoadMethodCache)
0076
0077 #define INLINE_CACHE_ENTRIES_STORE_ATTR CACHE_ENTRIES(_PyAttrCache)
0078
0079 typedef struct {
0080     uint16_t counter;
0081     uint16_t func_version[2];
0082 } _PyCallCache;
0083
0084 #define INLINE_CACHE_ENTRIES_CALL CACHE_ENTRIES(_PyCallCache)
0085
0086 typedef struct {
0087     uint16_t counter;
0088 } _PyStoreSubscrCache;
0089
0090 #define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache)
0091
0092 typedef struct {
0093     uint16_t counter;
0094 } _PyForIterCache;
0095
0096 #define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache)
0097
0098 typedef struct {
0099     uint16_t counter;
0100 } _PySendCache;
0101
0102 #define INLINE_CACHE_ENTRIES_SEND CACHE_ENTRIES(_PySendCache)
0103
0104 // Borrowed references to common callables:
0105 struct callable_cache {
0106     PyObject *isinstance;
0107     PyObject *len;
0108     PyObject *list_append;
0109     PyObject *object__getattribute__;
0110 };
0111
0112 /* "Locals plus" for a code object is the set of locals + cell vars +
0113  * free vars.  This relates to variable names as well as offsets into
0114  * the "fast locals" storage array of execution frames.  The compiler
0115  * builds the list of names, their offsets, and the corresponding
0116  * kind of local.
0117  *
0118  * Those kinds represent the source of the initial value and the
0119  * variable's scope (as related to closures).  A "local" is an
0120  * argument or other variable defined in the current scope.  A "free"
0121  * variable is one that is defined in an outer scope and comes from
0122  * the function's closure.  A "cell" variable is a local that escapes
0123  * into an inner function as part of a closure, and thus must be
0124  * wrapped in a cell.  Any "local" can also be a "cell", but the
0125  * "free" kind is mutually exclusive with both.
0126  */
0127
0128 // Note that these all fit within a byte, as do combinations.
0129 // Later, we will use the smaller numbers to differentiate the different
0130 // kinds of locals (e.g. pos-only arg, varkwargs, local-only).
0131 #define CO_FAST_HIDDEN  0x10
0132 #define CO_FAST_LOCAL   0x20
0133 #define CO_FAST_CELL    0x40
0134 #define CO_FAST_FREE    0x80
0135
0136 typedef unsigned char _PyLocals_Kind;
0137
0138 static inline _PyLocals_Kind
0139 _PyLocals_GetKind(PyObject *kinds, int i)
0140 {
0141     assert(PyBytes_Check(kinds));
0142     assert(0 <= i && i < PyBytes_GET_SIZE(kinds));
0143     char *ptr = PyBytes_AS_STRING(kinds);
0144     return (_PyLocals_Kind)(ptr[i]);
0145 }
0146
0147 static inline void
0148 _PyLocals_SetKind(PyObject *kinds, int i, _PyLocals_Kind kind)
0149 {
0150     assert(PyBytes_Check(kinds));
0151     assert(0 <= i && i < PyBytes_GET_SIZE(kinds));
0152     char *ptr = PyBytes_AS_STRING(kinds);
0153     ptr[i] = (char) kind;
0154 }
0155
0156
0157 struct _PyCodeConstructor {
0158     /* metadata */
0159     PyObject *filename;
0160     PyObject *name;
0161     PyObject *qualname;
0162     int flags;
0163
0164     /* the code */
0165     PyObject *code;
0166     int firstlineno;
0167     PyObject *linetable;
0168
0169     /* used by the code */
0170     PyObject *consts;
0171     PyObject *names;
0172
0173     /* mapping frame offsets to information */
0174     PyObject *localsplusnames;  // Tuple of strings
0175     PyObject *localspluskinds;  // Bytes object, one byte per variable
0176
0177     /* args (within varnames) */
0178     int argcount;
0179     int posonlyargcount;
0180     // XXX Replace argcount with posorkwargcount (argcount - posonlyargcount).
0181     int kwonlyargcount;
0182
0183     /* needed to create the frame */
0184     int stacksize;
0185
0186     /* used by the eval loop */
0187     PyObject *exceptiontable;
0188 };
0189
0190 // Using an "arguments struct" like this is helpful for maintainability
0191 // in a case such as this with many parameters.  It does bear a risk:
0192 // if the struct changes and callers are not updated properly then the
0193 // compiler will not catch problems (like a missing argument).  This can
0194 // cause hard-to-debug problems.  The risk is mitigated by the use of
0195 // check_code() in codeobject.c.  However, we may decide to switch
0196 // back to a regular function signature.  Regardless, this approach
0197 // wouldn't be appropriate if this weren't a strictly internal API.
0198 // (See the comments in https://github.com/python/cpython/pull/26258.)
0199 PyAPI_FUNC(int) _PyCode_Validate(struct _PyCodeConstructor *);
0200 PyAPI_FUNC(PyCodeObject *) _PyCode_New(struct _PyCodeConstructor *);
0201
0202
0203 /* Private API */
0204
0205 /* Getters for internal PyCodeObject data. */
0206 extern PyObject* _PyCode_GetVarnames(PyCodeObject *);
0207 extern PyObject* _PyCode_GetCellvars(PyCodeObject *);
0208 extern PyObject* _PyCode_GetFreevars(PyCodeObject *);
0209 extern PyObject* _PyCode_GetCode(PyCodeObject *);
0210
0211 /** API for initializing the line number tables. */
0212 extern int _PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds);
0213
0214 /** Out of process API for initializing the location table. */
0215 extern void _PyLineTable_InitAddressRange(
0216     const char *linetable,
0217     Py_ssize_t length,
0218     int firstlineno,
0219     PyCodeAddressRange *range);
0220
0221 /** API for traversing the line number table. */
0222 extern int _PyLineTable_NextAddressRange(PyCodeAddressRange *range);
0223 extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range);
0224
0225 /* Specialization functions */
0226
0227 extern void _Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls,
0228                                          _Py_CODEUNIT *instr, int load_method);
0229 extern void _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr,
0230                                     PyObject *name);
0231 extern void _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr,
0232                                      PyObject *name);
0233 extern void _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins,
0234                                       _Py_CODEUNIT *instr, PyObject *name);
0235 extern void _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container,
0236                                         _Py_CODEUNIT *instr);
0237 extern void _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub,
0238                                        _Py_CODEUNIT *instr);
0239 extern void _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr,
0240                                 int nargs, PyObject *kwnames);
0241 extern void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
0242                                     int oparg, PyObject **locals);
0243 extern void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs,
0244                                      _Py_CODEUNIT *instr, int oparg);
0245 extern void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr,
0246                                           int oparg);
0247 extern void _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg);
0248 extern void _Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr);
0249
0250 /* Finalizer function for static codeobjects used in deepfreeze.py */
0251 extern void _PyStaticCode_Fini(PyCodeObject *co);
0252 /* Function to intern strings of codeobjects and quicken the bytecode */
0253 extern int _PyStaticCode_Init(PyCodeObject *co);
0254
0255 #ifdef Py_STATS
0256
0257
0258 #define STAT_INC(opname, name) do { if (_py_stats) _py_stats->opcode_stats[opname].specialization.name++; } while (0)
0259 #define STAT_DEC(opname, name) do { if (_py_stats) _py_stats->opcode_stats[opname].specialization.name--; } while (0)
0260 #define OPCODE_EXE_INC(opname) do { if (_py_stats) _py_stats->opcode_stats[opname].execution_count++; } while (0)
0261 #define CALL_STAT_INC(name) do { if (_py_stats) _py_stats->call_stats.name++; } while (0)
0262 #define OBJECT_STAT_INC(name) do { if (_py_stats) _py_stats->object_stats.name++; } while (0)
0263 #define OBJECT_STAT_INC_COND(name, cond) \
0264     do { if (_py_stats && cond) _py_stats->object_stats.name++; } while (0)
0265 #define EVAL_CALL_STAT_INC(name) do { if (_py_stats) _py_stats->call_stats.eval_calls[name]++; } while (0)
0266 #define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) \
0267     do { if (_py_stats && PyFunction_Check(callable)) _py_stats->call_stats.eval_calls[name]++; } while (0)
0268
0269 // Used by the _opcode extension which is built as a shared library
0270 PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void);
0271
0272 #else
0273 #define STAT_INC(opname, name) ((void)0)
0274 #define STAT_DEC(opname, name) ((void)0)
0275 #define OPCODE_EXE_INC(opname) ((void)0)
0276 #define CALL_STAT_INC(name) ((void)0)
0277 #define OBJECT_STAT_INC(name) ((void)0)
0278 #define OBJECT_STAT_INC_COND(name, cond) ((void)0)
0279 #define EVAL_CALL_STAT_INC(name) ((void)0)
0280 #define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) ((void)0)
0281 #endif  // !Py_STATS
0282
0283 // Utility functions for reading/writing 32/64-bit values in the inline caches.
0284 // Great care should be taken to ensure that these functions remain correct and
0285 // performant! They should compile to just "move" instructions on all supported
0286 // compilers and platforms.
0287
0288 // We use memcpy to let the C compiler handle unaligned accesses and endianness
0289 // issues for us. It also seems to produce better code than manual copying for
0290 // most compilers (see https://blog.regehr.org/archives/959 for more info).
0291
0292 static inline void
0293 write_u32(uint16_t *p, uint32_t val)
0294 {
0295     memcpy(p, &val, sizeof(val));
0296 }
0297
0298 static inline void
0299 write_u64(uint16_t *p, uint64_t val)
0300 {
0301     memcpy(p, &val, sizeof(val));
0302 }
0303
0304 static inline void
0305 write_obj(uint16_t *p, PyObject *val)
0306 {
0307     memcpy(p, &val, sizeof(val));
0308 }
0309
0310 static inline uint16_t
0311 read_u16(uint16_t *p)
0312 {
0313     return *p;
0314 }
0315
0316 static inline uint32_t
0317 read_u32(uint16_t *p)
0318 {
0319     uint32_t val;
0320     memcpy(&val, p, sizeof(val));
0321     return val;
0322 }
0323
0324 static inline uint64_t
0325 read_u64(uint16_t *p)
0326 {
0327     uint64_t val;
0328     memcpy(&val, p, sizeof(val));
0329     return val;
0330 }
0331
0332 static inline PyObject *
0333 read_obj(uint16_t *p)
0334 {
0335     PyObject *val;
0336     memcpy(&val, p, sizeof(val));
0337     return val;
0338 }
0339
0340 /* See Objects/exception_handling_notes.txt for details.
0341  */
0342 static inline unsigned char *
0343 parse_varint(unsigned char *p, int *result) {
0344     int val = p[0] & 63;
0345     while (p[0] & 64) {
0346         p++;
0347         val = (val << 6) | (p[0] & 63);
0348     }
0349     *result = val;
0350     return p+1;
0351 }
0352
0353 static inline int
0354 write_varint(uint8_t *ptr, unsigned int val)
0355 {
0356     int written = 1;
0357     while (val >= 64) {
0358         *ptr++ = 64 | (val & 63);
0359         val >>= 6;
0360         written++;
0361     }
0362     *ptr = (uint8_t)val;
0363     return written;
0364 }
0365
0366 static inline int
0367 write_signed_varint(uint8_t *ptr, int val)
0368 {
0369     unsigned int uval;
0370     if (val < 0) {
0371         // (unsigned int)(-val) has an undefined behavior for INT_MIN
0372         uval = ((0 - (unsigned int)val) << 1) | 1;
0373     }
0374     else {
0375         uval = (unsigned int)val << 1;
0376     }
0377     return write_varint(ptr, uval);
0378 }
0379
0380 static inline int
0381 write_location_entry_start(uint8_t *ptr, int code, int length)
0382 {
0383     assert((code & 15) == code);
0384     *ptr = 128 | (uint8_t)(code << 3) | (uint8_t)(length - 1);
0385     return 1;
0386 }
0387
0388
0389 /** Counters
0390  * The first 16-bit value in each inline cache is a counter.
0391  * When counting misses, the counter is treated as a simple unsigned value.
0392  *
0393  * When counting executions until the next specialization attempt,
0394  * exponential backoff is used to reduce the number of specialization failures.
0395  * The high 12 bits store the counter, the low 4 bits store the backoff exponent.
0396  * On a specialization failure, the backoff exponent is incremented and the
0397  * counter set to (2**backoff - 1).
0398  * Backoff == 6 -> starting counter == 63, backoff == 10 -> starting counter == 1023.
0399  */
0400
0401 /* With a 16-bit counter, we have 12 bits for the counter value, and 4 bits for the backoff */
0402 #define ADAPTIVE_BACKOFF_BITS 4
0403
0404 // A value of 1 means that we attempt to specialize the *second* time each
0405 // instruction is executed. Executing twice is a much better indicator of
0406 // "hotness" than executing once, but additional warmup delays only prevent
0407 // specialization. Most types stabilize by the second execution, too:
0408 #define ADAPTIVE_WARMUP_VALUE 1
0409 #define ADAPTIVE_WARMUP_BACKOFF 1
0410
0411 // A value of 52 means that we attempt to re-specialize after 53 misses (a prime
0412 // number, useful for avoiding artifacts if every nth value is a different type
0413 // or something). Setting the backoff to 0 means that the counter is reset to
0414 // the same state as a warming-up instruction (value == 1, backoff == 1) after
0415 // deoptimization. This isn't strictly necessary, but it is bit easier to reason
0416 // about when thinking about the opcode transitions as a state machine:
0417 #define ADAPTIVE_COOLDOWN_VALUE 52
0418 #define ADAPTIVE_COOLDOWN_BACKOFF 0
0419
0420 #define MAX_BACKOFF_VALUE (16 - ADAPTIVE_BACKOFF_BITS)
0421
0422
0423 static inline uint16_t
0424 adaptive_counter_bits(uint16_t value, uint16_t backoff) {
0425     return ((value << ADAPTIVE_BACKOFF_BITS)
0426             | (backoff & ((1 << ADAPTIVE_BACKOFF_BITS) - 1)));
0427 }
0428
0429 static inline uint16_t
0430 adaptive_counter_warmup(void) {
0431     return adaptive_counter_bits(ADAPTIVE_WARMUP_VALUE,
0432                                  ADAPTIVE_WARMUP_BACKOFF);
0433 }
0434
0435 static inline uint16_t
0436 adaptive_counter_cooldown(void) {
0437     return adaptive_counter_bits(ADAPTIVE_COOLDOWN_VALUE,
0438                                  ADAPTIVE_COOLDOWN_BACKOFF);
0439 }
0440
0441 static inline uint16_t
0442 adaptive_counter_backoff(uint16_t counter) {
0443     uint16_t backoff = counter & ((1 << ADAPTIVE_BACKOFF_BITS) - 1);
0444     backoff++;
0445     if (backoff > MAX_BACKOFF_VALUE) {
0446         backoff = MAX_BACKOFF_VALUE;
0447     }
0448     uint16_t value = (uint16_t)(1 << backoff) - 1;
0449     return adaptive_counter_bits(value, backoff);
0450 }
0451
0452
0453 /* Line array cache for tracing */
0454
0455 typedef struct _PyShimCodeDef {
0456     const uint8_t *code;
0457     int codelen;
0458     int stacksize;
0459     const char *cname;
0460 } _PyShimCodeDef;
0461
0462 extern PyCodeObject *
0463 _Py_MakeShimCode(const _PyShimCodeDef *code);
0464
0465 extern uint32_t _Py_next_func_version;
0466
0467
0468 /* Comparison bit masks. */
0469
0470 /* Note this evaluates its arguments twice each */
0471 #define COMPARISON_BIT(x, y) (1 << (2 * ((x) >= (y)) + ((x) <= (y))))
0472
0473 /*
0474  * The following bits are chosen so that the value of
0475  * COMPARSION_BIT(left, right)
0476  * masked by the values below will be non-zero if the
0477  * comparison is true, and zero if it is false */
0478
0479 /* This is for values that are unordered, ie. NaN, not types that are unordered, e.g. sets */
0480 #define COMPARISON_UNORDERED 1
0481
0482 #define COMPARISON_LESS_THAN 2
0483 #define COMPARISON_GREATER_THAN 4
0484 #define COMPARISON_EQUALS 8
0485
0486 #define COMPARISON_NOT_EQUALS (COMPARISON_UNORDERED | COMPARISON_LESS_THAN | COMPARISON_GREATER_THAN)
0487
0488 extern int _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp);
0489
0490 extern int _Py_GetBaseOpcode(PyCodeObject *code, int offset);
0491
0492
0493 #ifdef __cplusplus
0494 }
0495 #endif
0496 #endif /* !Py_INTERNAL_CODE_H */