Skip to content

Commit e117c02

Browse files
authored
bpo-44337: Port LOAD_ATTR to PEP 659 adaptive interpreter (GH-26595)
* Specialize LOAD_ATTR with LOAD_ATTR_SLOT and LOAD_ATTR_SPLIT_KEYS * Move dict-common.h to internal/pycore_dict.h * Add LOAD_ATTR_WITH_HINT specialized opcode. * Quicken in function if loopy * Specialize LOAD_ATTR for module attributes. * Add specialization stats
1 parent 309ab61 commit e117c02

File tree

14 files changed

+508
-218
lines changed

14 files changed

+508
-218
lines changed

Include/internal/pycore_code.h

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,12 @@ typedef struct {
4242
uint16_t index;
4343
} _PyAdaptiveEntry;
4444

45+
46+
typedef struct {
47+
uint32_t tp_version;
48+
uint32_t dk_version_or_hint;
49+
} _PyLoadAttrCache;
50+
4551
/* Add specialized versions of entries to this union.
4652
*
4753
* Do not break the invariant: sizeof(SpecializedCacheEntry) == 8
@@ -55,6 +61,7 @@ typedef struct {
5561
typedef union {
5662
_PyEntryZero zero;
5763
_PyAdaptiveEntry adaptive;
64+
_PyLoadAttrCache load_attr;
5865
} SpecializedCacheEntry;
5966

6067
#define INSTRUCTIONS_PER_ENTRY (sizeof(SpecializedCacheEntry)/sizeof(_Py_CODEUNIT))
@@ -255,6 +262,83 @@ PyAPI_FUNC(PyObject *) _PyCode_GetCellvars(PyCodeObject *);
255262
PyAPI_FUNC(PyObject *) _PyCode_GetFreevars(PyCodeObject *);
256263

257264

265+
/* Cache hits and misses */
266+
267+
static inline uint8_t
268+
saturating_increment(uint8_t c)
269+
{
270+
return c<<1;
271+
}
272+
273+
static inline uint8_t
274+
saturating_decrement(uint8_t c)
275+
{
276+
return (c>>1) + 128;
277+
}
278+
279+
static inline uint8_t
280+
saturating_zero(void)
281+
{
282+
return 255;
283+
}
284+
285+
/* Starting value for saturating counter.
286+
* Technically this should be 1, but that is likely to
287+
* cause a bit of thrashing when we optimize then get an immediate miss.
288+
* We want to give the counter a change to stabilize, so we start at 3.
289+
*/
290+
static inline uint8_t
291+
saturating_start(void)
292+
{
293+
return saturating_zero()<<3;
294+
}
295+
296+
static inline void
297+
record_cache_hit(_PyAdaptiveEntry *entry) {
298+
entry->counter = saturating_increment(entry->counter);
299+
}
300+
301+
static inline void
302+
record_cache_miss(_PyAdaptiveEntry *entry) {
303+
entry->counter = saturating_decrement(entry->counter);
304+
}
305+
306+
static inline int
307+
too_many_cache_misses(_PyAdaptiveEntry *entry) {
308+
return entry->counter == saturating_zero();
309+
}
310+
311+
#define BACKOFF 64
312+
313+
static inline void
314+
cache_backoff(_PyAdaptiveEntry *entry) {
315+
entry->counter = BACKOFF;
316+
}
317+
318+
/* Specialization functions */
319+
320+
int _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache);
321+
322+
#define SPECIALIZATION_STATS 0
323+
#if SPECIALIZATION_STATS
324+
325+
typedef struct _specialization_stats {
326+
uint64_t specialization_success;
327+
uint64_t specialization_failure;
328+
uint64_t loadattr_hit;
329+
uint64_t loadattr_deferred;
330+
uint64_t loadattr_miss;
331+
uint64_t loadattr_deopt;
332+
} SpecializationStats;
333+
334+
extern SpecializationStats _specialization_stats;
335+
#define STAT_INC(name) _specialization_stats.name++
336+
void _Py_PrintSpecializationStats(void);
337+
#else
338+
#define STAT_INC(name) ((void)0)
339+
#endif
340+
341+
258342
#ifdef __cplusplus
259343
}
260344
#endif

Objects/dict-common.h renamed to Include/internal/pycore_dict.h

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,14 @@
1-
#ifndef Py_DICT_COMMON_H
2-
#define Py_DICT_COMMON_H
1+
2+
#ifndef Py_INTERNAL_DICT_H
3+
#define Py_INTERNAL_DICT_H
4+
#ifdef __cplusplus
5+
extern "C" {
6+
#endif
7+
8+
#ifndef Py_BUILD_CORE
9+
# error "this header requires Py_BUILD_CORE define"
10+
#endif
11+
312

413
typedef struct {
514
/* Cached hash code of me_key. */
@@ -62,4 +71,26 @@ struct _dictkeysobject {
6271
see the DK_ENTRIES() macro */
6372
};
6473

74+
#define DK_LOG_SIZE(dk) ((dk)->dk_log2_size)
75+
#if SIZEOF_VOID_P > 4
76+
#define DK_SIZE(dk) (((int64_t)1)<<DK_LOG_SIZE(dk))
77+
#define DK_IXSIZE(dk) \
78+
(DK_LOG_SIZE(dk) <= 7 ? \
79+
1 : DK_LOG_SIZE(dk) <= 15 ? \
80+
2 : DK_LOG_SIZE(dk) <= 31 ? \
81+
4 : sizeof(int64_t))
82+
#else
83+
#define DK_SIZE(dk) (1<<DK_LOG_SIZE(dk))
84+
#define DK_IXSIZE(dk) \
85+
(DK_LOG_SIZE(dk) <= 7 ? \
86+
1 : DK_LOG_SIZE(dk) <= 15 ? \
87+
2 : sizeof(int32_t))
88+
#endif
89+
#define DK_ENTRIES(dk) \
90+
((PyDictKeyEntry*)(&((int8_t*)((dk)->dk_indices))[DK_SIZE(dk) * DK_IXSIZE(dk)]))
91+
92+
93+
#ifdef __cplusplus
94+
}
6595
#endif
96+
#endif /* !Py_INTERNAL_DICT_H */

Include/opcode.h

Lines changed: 6 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Lib/opcode.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,3 +218,12 @@ def jabs_op(name, op):
218218
def_op('CALL_METHOD_KW', 166)
219219

220220
del def_op, name_op, jrel_op, jabs_op
221+
222+
_specialized_instructions = [
223+
"JUMP_ABSOLUTE_QUICK",
224+
"LOAD_ATTR_ADAPTIVE",
225+
"LOAD_ATTR_SPLIT_KEYS",
226+
"LOAD_ATTR_WITH_HINT",
227+
"LOAD_ATTR_SLOT",
228+
"LOAD_ATTR_MODULE",
229+
]

Lib/test/test_capi.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -323,7 +323,7 @@ class C(): pass
323323
break
324324
"""
325325
rc, out, err = assert_python_ok('-c', code)
326-
self.assertIn(b'MemoryError 1 10', out)
326+
self.assertIn(b'MemoryError 1', out)
327327
self.assertIn(b'MemoryError 2 20', out)
328328
self.assertIn(b'MemoryError 3 30', out)
329329

Makefile.pre.in

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -979,8 +979,7 @@ Objects/bytearrayobject.o: $(srcdir)/Objects/bytearrayobject.c $(BYTESTR_DEPS)
979979

980980
Objects/unicodeobject.o: $(srcdir)/Objects/unicodeobject.c $(UNICODE_DEPS)
981981

982-
Objects/odictobject.o: $(srcdir)/Objects/dict-common.h
983-
Objects/dictobject.o: $(srcdir)/Objects/stringlib/eq.h $(srcdir)/Objects/dict-common.h
982+
Objects/dictobject.o: $(srcdir)/Objects/stringlib/eq.h
984983
Objects/setobject.o: $(srcdir)/Objects/stringlib/eq.h
985984

986985
.PHONY: regen-opcode-targets
@@ -1156,6 +1155,7 @@ PYTHON_HEADERS= \
11561155
$(srcdir)/Include/internal/pycore_compile.h \
11571156
$(srcdir)/Include/internal/pycore_condvar.h \
11581157
$(srcdir)/Include/internal/pycore_context.h \
1158+
$(srcdir)/Include/internal/pycore_dict.h \
11591159
$(srcdir)/Include/internal/pycore_dtoa.h \
11601160
$(srcdir)/Include/internal/pycore_fileutils.h \
11611161
$(srcdir)/Include/internal/pycore_format.h \
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
Initial implementation of adaptive specialization of LOAD_ATTR
2+
3+
Four specialized forms of LOAD_ATTR are added:
4+
5+
* LOAD_ATTR_SLOT
6+
7+
* LOAD_ATTR_SPLIT_KEYS
8+
9+
* LOAD_ATTR_WITH_HINT
10+
11+
* LOAD_ATTR_MODULE

Objects/dictobject.c

Lines changed: 2 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ converting the dict to the combined table.
117117
#include "pycore_object.h" // _PyObject_GC_TRACK()
118118
#include "pycore_pyerrors.h" // _PyErr_Fetch()
119119
#include "pycore_pystate.h" // _PyThreadState_GET()
120-
#include "dict-common.h"
120+
#include "pycore_dict.h"
121121
#include "stringlib/eq.h" // unicode_eq()
122122

123123
/*[clinic input]
@@ -285,24 +285,6 @@ _PyDict_DebugMallocStats(FILE *out)
285285
state->numfree, sizeof(PyDictObject));
286286
}
287287

288-
#define DK_LOG_SIZE(dk) ((dk)->dk_log2_size)
289-
#if SIZEOF_VOID_P > 4
290-
#define DK_SIZE(dk) (((int64_t)1)<<DK_LOG_SIZE(dk))
291-
#define DK_IXSIZE(dk) \
292-
(DK_LOG_SIZE(dk) <= 7 ? \
293-
1 : DK_LOG_SIZE(dk) <= 15 ? \
294-
2 : DK_LOG_SIZE(dk) <= 31 ? \
295-
4 : sizeof(int64_t))
296-
#else
297-
#define DK_SIZE(dk) (1<<DK_LOG_SIZE(dk))
298-
#define DK_IXSIZE(dk) \
299-
(DK_LOG_SIZE(dk) <= 7 ? \
300-
1 : DK_LOG_SIZE(dk) <= 15 ? \
301-
2 : sizeof(int32_t))
302-
#endif
303-
#define DK_ENTRIES(dk) \
304-
((PyDictKeyEntry*)(&((int8_t*)((dk)->dk_indices))[DK_SIZE(dk) * DK_IXSIZE(dk)]))
305-
306288
#define DK_MASK(dk) (DK_SIZE(dk)-1)
307289
#define IS_POWER_OF_2(x) (((x) & (x-1)) == 0)
308290

@@ -1544,10 +1526,10 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix,
15441526
assert(hashpos >= 0);
15451527

15461528
mp->ma_used--;
1529+
mp->ma_keys->dk_version = 0;
15471530
mp->ma_version_tag = DICT_NEXT_VERSION();
15481531
ep = &DK_ENTRIES(mp->ma_keys)[ix];
15491532
dictkeys_set_index(mp->ma_keys, hashpos, DKIX_DUMMY);
1550-
mp->ma_keys->dk_version = 0;
15511533
old_key = ep->me_key;
15521534
ep->me_key = NULL;
15531535
ep->me_value = NULL;

Objects/odictobject.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -467,7 +467,7 @@ Potential Optimizations
467467
#include "Python.h"
468468
#include "pycore_object.h"
469469
#include <stddef.h> // offsetof()
470-
#include "dict-common.h"
470+
#include "pycore_dict.h"
471471
#include <stddef.h>
472472

473473
#include "clinic/odictobject.c.h"

0 commit comments

Comments
 (0)