Skip to content

Commit 3b3d30e

Browse files
sweeneydesweeneyde
authored
bpo-45367: Specialize BINARY_MULTIPLY (GH-28727)
1 parent c96d154 commit 3b3d30e

File tree

9 files changed

+162
-55
lines changed

9 files changed

+162
-55
lines changed

‎Include/internal/pycore_code.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -307,7 +307,8 @@ int _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *nam
307307
int _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache);
308308
int _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache);
309309
int _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, _Py_CODEUNIT *instr);
310-
int _Py_Specialize_BinaryAdd(PyObject *sub, PyObject *container, _Py_CODEUNIT *instr);
310+
int _Py_Specialize_BinaryAdd(PyObject *left, PyObject *right, _Py_CODEUNIT *instr);
311+
int _Py_Specialize_BinaryMultiply(PyObject *left, PyObject *right, _Py_CODEUNIT *instr);
311312

312313
#define PRINT_SPECIALIZATION_STATS 0
313314
#define PRINT_SPECIALIZATION_STATS_DETAILED 0

‎Include/internal/pycore_long.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ static inline PyObject* _PyLong_GetOne(void)
3535
{ return __PyLong_GetSmallInt_internal(1); }
3636

3737
PyObject *_PyLong_Add(PyLongObject *left, PyLongObject *right);
38+
PyObject *_PyLong_Multiply(PyLongObject *left, PyLongObject *right);
3839

3940
#ifdef __cplusplus
4041
}

‎Include/opcode.h‎

Lines changed: 30 additions & 27 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Lib/opcode.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,9 @@ def jabs_op(name, op):
225225
"BINARY_ADD_FLOAT",
226226
"BINARY_ADD_UNICODE",
227227
"BINARY_ADD_UNICODE_INPLACE_FAST",
228+
"BINARY_MULTIPLY_ADAPTIVE",
229+
"BINARY_MULTIPLY_INT",
230+
"BINARY_MULTIPLY_FLOAT",
228231
"BINARY_SUBSCR_ADAPTIVE",
229232
"BINARY_SUBSCR_LIST_INT",
230233
"BINARY_SUBSCR_TUPLE_INT",

‎Misc/NEWS.d/next/Core and Builtins/2021-10-05-03-49-07.bpo-45367._astoU.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Specialized the ``BINARY_MULTIPLY`` opcode to ``BINARY_MULTIPLY_INT`` and ``BINARY_MULTIPLY_FLOAT`` using the PEP 659 machinery.

‎Objects/longobject.c‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3594,13 +3594,11 @@ k_lopsided_mul(PyLongObject *a, PyLongObject *b)
35943594
return NULL;
35953595
}
35963596

3597-
static PyObject *
3598-
long_mul(PyLongObject *a, PyLongObject *b)
3597+
PyObject *
3598+
_PyLong_Multiply(PyLongObject *a, PyLongObject *b)
35993599
{
36003600
PyLongObject *z;
36013601

3602-
CHECK_BINOP(a, b);
3603-
36043602
/* fast path for single-digit multiplication */
36053603
if (IS_MEDIUM_VALUE(a) && IS_MEDIUM_VALUE(b)) {
36063604
stwodigits v = medium_value(a) * medium_value(b);
@@ -3617,6 +3615,13 @@ long_mul(PyLongObject *a, PyLongObject *b)
36173615
return (PyObject *)z;
36183616
}
36193617

3618+
static PyObject *
3619+
long_mul(PyLongObject *a, PyLongObject *b)
3620+
{
3621+
CHECK_BINOP(a, b);
3622+
return _PyLong_Multiply(a, b);
3623+
}
3624+
36203625
/* Fast modulo division for single-digit longs. */
36213626
static PyObject *
36223627
fast_mod(PyLongObject *a, PyLongObject *b)

‎Python/ceval.c‎

Lines changed: 61 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1933,14 +1933,73 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
19331933
}
19341934

19351935
TARGET(BINARY_MULTIPLY) {
1936+
PREDICTED(BINARY_MULTIPLY);
1937+
STAT_INC(BINARY_MULTIPLY, unquickened);
19361938
PyObject *right = POP();
19371939
PyObject *left = TOP();
19381940
PyObject *res = PyNumber_Multiply(left, right);
19391941
Py_DECREF(left);
19401942
Py_DECREF(right);
19411943
SET_TOP(res);
1942-
if (res == NULL)
1944+
if (res == NULL) {
19431945
goto error;
1946+
}
1947+
DISPATCH();
1948+
}
1949+
1950+
TARGET(BINARY_MULTIPLY_ADAPTIVE) {
1951+
if (oparg == 0) {
1952+
PyObject *left = SECOND();
1953+
PyObject *right = TOP();
1954+
next_instr--;
1955+
if (_Py_Specialize_BinaryMultiply(left, right, next_instr) < 0) {
1956+
goto error;
1957+
}
1958+
DISPATCH();
1959+
}
1960+
else {
1961+
STAT_INC(BINARY_MULTIPLY, deferred);
1962+
UPDATE_PREV_INSTR_OPARG(next_instr, oparg - 1);
1963+
STAT_DEC(BINARY_MULTIPLY, unquickened);
1964+
JUMP_TO_INSTRUCTION(BINARY_MULTIPLY);
1965+
}
1966+
}
1967+
1968+
TARGET(BINARY_MULTIPLY_INT) {
1969+
PyObject *left = SECOND();
1970+
PyObject *right = TOP();
1971+
DEOPT_IF(!PyLong_CheckExact(left), BINARY_MULTIPLY);
1972+
DEOPT_IF(!PyLong_CheckExact(right), BINARY_MULTIPLY);
1973+
STAT_INC(BINARY_MULTIPLY, hit);
1974+
record_hit_inline(next_instr, oparg);
1975+
PyObject *prod = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right);
1976+
SET_SECOND(prod);
1977+
Py_DECREF(right);
1978+
Py_DECREF(left);
1979+
STACK_SHRINK(1);
1980+
if (prod == NULL) {
1981+
goto error;
1982+
}
1983+
DISPATCH();
1984+
}
1985+
1986+
TARGET(BINARY_MULTIPLY_FLOAT) {
1987+
PyObject *left = SECOND();
1988+
PyObject *right = TOP();
1989+
DEOPT_IF(!PyFloat_CheckExact(left), BINARY_MULTIPLY);
1990+
DEOPT_IF(!PyFloat_CheckExact(right), BINARY_MULTIPLY);
1991+
STAT_INC(BINARY_MULTIPLY, hit);
1992+
record_hit_inline(next_instr, oparg);
1993+
double dprod = ((PyFloatObject *)left)->ob_fval *
1994+
((PyFloatObject *)right)->ob_fval;
1995+
PyObject *prod = PyFloat_FromDouble(dprod);
1996+
SET_SECOND(prod);
1997+
Py_DECREF(right);
1998+
Py_DECREF(left);
1999+
STACK_SHRINK(1);
2000+
if (prod == NULL) {
2001+
goto error;
2002+
}
19442003
DISPATCH();
19452004
}
19462005

@@ -4954,6 +5013,7 @@ MISS_WITH_CACHE(LOAD_GLOBAL)
49545013
MISS_WITH_CACHE(LOAD_METHOD)
49555014
MISS_WITH_OPARG_COUNTER(BINARY_SUBSCR)
49565015
MISS_WITH_OPARG_COUNTER(BINARY_ADD)
5016+
MISS_WITH_OPARG_COUNTER(BINARY_MULTIPLY)
49575017

49585018
binary_subscr_dict_error:
49595019
{

‎Python/opcode_targets.h‎

Lines changed: 22 additions & 22 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Python/specialize.c‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ _Py_GetSpecializationStats(void) {
124124
err += add_stat_dict(stats, LOAD_GLOBAL, "load_global");
125125
err += add_stat_dict(stats, LOAD_METHOD, "load_method");
126126
err += add_stat_dict(stats, BINARY_ADD, "binary_add");
127+
err += add_stat_dict(stats, BINARY_MULTIPLY, "binary_multiply");
127128
err += add_stat_dict(stats, BINARY_SUBSCR, "binary_subscr");
128129
err += add_stat_dict(stats, STORE_ATTR, "store_attr");
129130
if (err < 0) {
@@ -180,6 +181,7 @@ _Py_PrintSpecializationStats(void)
180181
print_stats(out, &_specialization_stats[LOAD_GLOBAL], "load_global");
181182
print_stats(out, &_specialization_stats[LOAD_METHOD], "load_method");
182183
print_stats(out, &_specialization_stats[BINARY_ADD], "binary_add");
184+
print_stats(out, &_specialization_stats[BINARY_MULTIPLY], "binary_multiply");
183185
print_stats(out, &_specialization_stats[BINARY_SUBSCR], "binary_subscr");
184186
print_stats(out, &_specialization_stats[STORE_ATTR], "store_attr");
185187
if (out != stderr) {
@@ -230,6 +232,7 @@ static uint8_t adaptive_opcodes[256] = {
230232
[LOAD_GLOBAL] = LOAD_GLOBAL_ADAPTIVE,
231233
[LOAD_METHOD] = LOAD_METHOD_ADAPTIVE,
232234
[BINARY_ADD] = BINARY_ADD_ADAPTIVE,
235+
[BINARY_MULTIPLY] = BINARY_MULTIPLY_ADAPTIVE,
233236
[BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE,
234237
[STORE_ATTR] = STORE_ATTR_ADAPTIVE,
235238
};
@@ -240,6 +243,7 @@ static uint8_t cache_requirements[256] = {
240243
[LOAD_GLOBAL] = 2, /* _PyAdaptiveEntry and _PyLoadGlobalCache */
241244
[LOAD_METHOD] = 3, /* _PyAdaptiveEntry, _PyAttrCache and _PyObjectCache */
242245
[BINARY_ADD] = 0,
246+
[BINARY_MULTIPLY] = 0,
243247
[BINARY_SUBSCR] = 0,
244248
[STORE_ATTR] = 2, /* _PyAdaptiveEntry and _PyAttrCache */
245249
};
@@ -1188,3 +1192,32 @@ _Py_Specialize_BinaryAdd(PyObject *left, PyObject *right, _Py_CODEUNIT *instr)
11881192
assert(!PyErr_Occurred());
11891193
return 0;
11901194
}
1195+
1196+
int
1197+
_Py_Specialize_BinaryMultiply(PyObject *left, PyObject *right, _Py_CODEUNIT *instr)
1198+
{
1199+
if (!Py_IS_TYPE(left, Py_TYPE(right))) {
1200+
SPECIALIZATION_FAIL(BINARY_MULTIPLY, SPEC_FAIL_DIFFERENT_TYPES);
1201+
goto fail;
1202+
}
1203+
if (PyLong_CheckExact(left)) {
1204+
*instr = _Py_MAKECODEUNIT(BINARY_MULTIPLY_INT, saturating_start());
1205+
goto success;
1206+
}
1207+
else if (PyFloat_CheckExact(left)) {
1208+
*instr = _Py_MAKECODEUNIT(BINARY_MULTIPLY_FLOAT, saturating_start());
1209+
goto success;
1210+
}
1211+
else {
1212+
SPECIALIZATION_FAIL(BINARY_MULTIPLY, SPEC_FAIL_OTHER);
1213+
}
1214+
fail:
1215+
STAT_INC(BINARY_MULTIPLY, specialization_failure);
1216+
assert(!PyErr_Occurred());
1217+
*instr = _Py_MAKECODEUNIT(_Py_OPCODE(*instr), ADAPTIVE_CACHE_BACKOFF);
1218+
return 0;
1219+
success:
1220+
STAT_INC(BINARY_MULTIPLY, specialization_success);
1221+
assert(!PyErr_Occurred());
1222+
return 0;
1223+
}

0 commit comments

Comments
 (0)