diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 188da775eb1cc7..6565910d848206 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -615,6 +615,25 @@ PyStackRef_FromPyObjectBorrow(PyObject *obj) return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT}; } +/* Create a stackref from a pre-tagged operand (tag bits already set). + Used by _LOAD_CONST_INLINE_BORROW variants where the operand is + tagged at trace creation time to avoid tagging on every execution. */ +static inline _PyStackRef +_PyStackRef_FromPreTagged(uintptr_t tagged) +{ + assert(tagged & Py_TAG_REFCNT); + return (_PyStackRef){ .bits = tagged }; +} +#define PyStackRef_FromPreTagged(ptr) _PyStackRef_FromPreTagged((uintptr_t)(ptr)) + +/* Tag a PyObject pointer as a borrowed operand for BORROW variants. */ +#define PyStackRef_TagBorrow(ptr) \ + ((uintptr_t)(ptr) | Py_TAG_REFCNT) + +/* Strip tag bits from a pre-tagged operand to recover the PyObject pointer. */ +#define PyStackRef_UntagBorrow(tagged) \ + ((PyObject *)((uintptr_t)(tagged) & ~Py_TAG_BITS)) + /* WARNING: This macro evaluates its argument more than once */ #ifdef _WIN32 #define PyStackRef_DUP(REF) \ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 5fcc422749a3d3..7865d9ecf7cfb9 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -943,443 +943,431 @@ extern "C" { #define _INSERT_1_LOAD_CONST_INLINE_r02 1151 #define _INSERT_1_LOAD_CONST_INLINE_r12 1152 #define _INSERT_1_LOAD_CONST_INLINE_r23 1153 -#define _INSERT_1_LOAD_CONST_INLINE_BORROW_r02 1154 -#define _INSERT_1_LOAD_CONST_INLINE_BORROW_r12 1155 -#define _INSERT_1_LOAD_CONST_INLINE_BORROW_r23 1156 -#define _INSERT_2_LOAD_CONST_INLINE_BORROW_r03 1157 -#define _INSERT_2_LOAD_CONST_INLINE_BORROW_r13 1158 -#define _INSERT_2_LOAD_CONST_INLINE_BORROW_r23 1159 -#define _INSERT_NULL_r10 1160 -#define _INSTRUMENTED_FOR_ITER_r23 1161 -#define _INSTRUMENTED_INSTRUCTION_r00 1162 -#define _INSTRUMENTED_JUMP_FORWARD_r00 1163 -#define _INSTRUMENTED_JUMP_FORWARD_r11 1164 -#define _INSTRUMENTED_JUMP_FORWARD_r22 1165 -#define _INSTRUMENTED_JUMP_FORWARD_r33 1166 -#define _INSTRUMENTED_LINE_r00 1167 -#define _INSTRUMENTED_NOT_TAKEN_r00 1168 -#define _INSTRUMENTED_NOT_TAKEN_r11 1169 -#define _INSTRUMENTED_NOT_TAKEN_r22 1170 -#define _INSTRUMENTED_NOT_TAKEN_r33 1171 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r00 1172 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 1173 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 1174 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 1175 -#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 1176 -#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 1177 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r00 1178 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 1179 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 1180 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 1181 -#define _IS_NONE_r11 1182 -#define _IS_OP_r03 1183 -#define _IS_OP_r13 1184 -#define _IS_OP_r23 1185 -#define _ITER_CHECK_LIST_r02 1186 -#define _ITER_CHECK_LIST_r12 1187 -#define _ITER_CHECK_LIST_r22 1188 -#define _ITER_CHECK_LIST_r33 1189 -#define _ITER_CHECK_RANGE_r02 1190 -#define _ITER_CHECK_RANGE_r12 1191 -#define _ITER_CHECK_RANGE_r22 1192 -#define _ITER_CHECK_RANGE_r33 1193 -#define _ITER_CHECK_TUPLE_r02 1194 -#define _ITER_CHECK_TUPLE_r12 1195 -#define _ITER_CHECK_TUPLE_r22 1196 -#define _ITER_CHECK_TUPLE_r33 1197 -#define _ITER_JUMP_LIST_r02 1198 -#define _ITER_JUMP_LIST_r12 1199 -#define _ITER_JUMP_LIST_r22 1200 -#define _ITER_JUMP_LIST_r33 1201 -#define _ITER_JUMP_RANGE_r02 1202 -#define _ITER_JUMP_RANGE_r12 1203 -#define _ITER_JUMP_RANGE_r22 1204 -#define _ITER_JUMP_RANGE_r33 1205 -#define _ITER_JUMP_TUPLE_r02 1206 -#define _ITER_JUMP_TUPLE_r12 1207 -#define _ITER_JUMP_TUPLE_r22 1208 -#define _ITER_JUMP_TUPLE_r33 1209 -#define _ITER_NEXT_LIST_r23 1210 -#define _ITER_NEXT_LIST_TIER_TWO_r23 1211 -#define _ITER_NEXT_RANGE_r03 1212 -#define _ITER_NEXT_RANGE_r13 1213 -#define _ITER_NEXT_RANGE_r23 1214 -#define _ITER_NEXT_TUPLE_r03 1215 -#define _ITER_NEXT_TUPLE_r13 1216 -#define _ITER_NEXT_TUPLE_r23 1217 -#define _JUMP_BACKWARD_NO_INTERRUPT_r00 1218 -#define _JUMP_BACKWARD_NO_INTERRUPT_r11 1219 -#define _JUMP_BACKWARD_NO_INTERRUPT_r22 1220 -#define _JUMP_BACKWARD_NO_INTERRUPT_r33 1221 -#define _JUMP_TO_TOP_r00 1222 -#define _LIST_APPEND_r10 1223 -#define _LIST_EXTEND_r11 1224 -#define _LOAD_ATTR_r10 1225 -#define _LOAD_ATTR_CLASS_r11 1226 -#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_r11 1227 -#define _LOAD_ATTR_INSTANCE_VALUE_r02 1228 -#define _LOAD_ATTR_INSTANCE_VALUE_r12 1229 -#define _LOAD_ATTR_INSTANCE_VALUE_r23 1230 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r02 1231 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 1232 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 1233 -#define _LOAD_ATTR_METHOD_NO_DICT_r02 1234 -#define _LOAD_ATTR_METHOD_NO_DICT_r12 1235 -#define _LOAD_ATTR_METHOD_NO_DICT_r23 1236 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r02 1237 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 1238 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 1239 -#define _LOAD_ATTR_MODULE_r12 1240 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 1241 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 1242 -#define _LOAD_ATTR_PROPERTY_FRAME_r11 1243 -#define _LOAD_ATTR_SLOT_r02 1244 -#define _LOAD_ATTR_SLOT_r12 1245 -#define _LOAD_ATTR_SLOT_r23 1246 -#define _LOAD_ATTR_WITH_HINT_r12 1247 -#define _LOAD_BUILD_CLASS_r01 1248 -#define _LOAD_BYTECODE_r00 1249 -#define _LOAD_COMMON_CONSTANT_r01 1250 -#define _LOAD_COMMON_CONSTANT_r12 1251 -#define _LOAD_COMMON_CONSTANT_r23 1252 -#define _LOAD_CONST_r01 1253 -#define _LOAD_CONST_r12 1254 -#define _LOAD_CONST_r23 1255 -#define _LOAD_CONST_INLINE_r01 1256 -#define _LOAD_CONST_INLINE_r12 1257 -#define _LOAD_CONST_INLINE_r23 1258 -#define _LOAD_CONST_INLINE_BORROW_r01 1259 -#define _LOAD_CONST_INLINE_BORROW_r12 1260 -#define _LOAD_CONST_INLINE_BORROW_r23 1261 -#define _LOAD_DEREF_r01 1262 -#define _LOAD_FAST_r01 1263 -#define _LOAD_FAST_r12 1264 -#define _LOAD_FAST_r23 1265 -#define _LOAD_FAST_0_r01 1266 -#define _LOAD_FAST_0_r12 1267 -#define _LOAD_FAST_0_r23 1268 -#define _LOAD_FAST_1_r01 1269 -#define _LOAD_FAST_1_r12 1270 -#define _LOAD_FAST_1_r23 1271 -#define _LOAD_FAST_2_r01 1272 -#define _LOAD_FAST_2_r12 1273 -#define _LOAD_FAST_2_r23 1274 -#define _LOAD_FAST_3_r01 1275 -#define _LOAD_FAST_3_r12 1276 -#define _LOAD_FAST_3_r23 1277 -#define _LOAD_FAST_4_r01 1278 -#define _LOAD_FAST_4_r12 1279 -#define _LOAD_FAST_4_r23 1280 -#define _LOAD_FAST_5_r01 1281 -#define _LOAD_FAST_5_r12 1282 -#define _LOAD_FAST_5_r23 1283 -#define _LOAD_FAST_6_r01 1284 -#define _LOAD_FAST_6_r12 1285 -#define _LOAD_FAST_6_r23 1286 -#define _LOAD_FAST_7_r01 1287 -#define _LOAD_FAST_7_r12 1288 -#define _LOAD_FAST_7_r23 1289 -#define _LOAD_FAST_AND_CLEAR_r01 1290 -#define _LOAD_FAST_AND_CLEAR_r12 1291 -#define _LOAD_FAST_AND_CLEAR_r23 1292 -#define _LOAD_FAST_BORROW_r01 1293 -#define _LOAD_FAST_BORROW_r12 1294 -#define _LOAD_FAST_BORROW_r23 1295 -#define _LOAD_FAST_BORROW_0_r01 1296 -#define _LOAD_FAST_BORROW_0_r12 1297 -#define _LOAD_FAST_BORROW_0_r23 1298 -#define _LOAD_FAST_BORROW_1_r01 1299 -#define _LOAD_FAST_BORROW_1_r12 1300 -#define _LOAD_FAST_BORROW_1_r23 1301 -#define _LOAD_FAST_BORROW_2_r01 1302 -#define _LOAD_FAST_BORROW_2_r12 1303 -#define _LOAD_FAST_BORROW_2_r23 1304 -#define _LOAD_FAST_BORROW_3_r01 1305 -#define _LOAD_FAST_BORROW_3_r12 1306 -#define _LOAD_FAST_BORROW_3_r23 1307 -#define _LOAD_FAST_BORROW_4_r01 1308 -#define _LOAD_FAST_BORROW_4_r12 1309 -#define _LOAD_FAST_BORROW_4_r23 1310 -#define _LOAD_FAST_BORROW_5_r01 1311 -#define _LOAD_FAST_BORROW_5_r12 1312 -#define _LOAD_FAST_BORROW_5_r23 1313 -#define _LOAD_FAST_BORROW_6_r01 1314 -#define _LOAD_FAST_BORROW_6_r12 1315 -#define _LOAD_FAST_BORROW_6_r23 1316 -#define _LOAD_FAST_BORROW_7_r01 1317 -#define _LOAD_FAST_BORROW_7_r12 1318 -#define _LOAD_FAST_BORROW_7_r23 1319 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1320 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1321 -#define _LOAD_FAST_CHECK_r01 1322 -#define _LOAD_FAST_CHECK_r12 1323 -#define _LOAD_FAST_CHECK_r23 1324 -#define _LOAD_FAST_LOAD_FAST_r02 1325 -#define _LOAD_FAST_LOAD_FAST_r13 1326 -#define _LOAD_FROM_DICT_OR_DEREF_r11 1327 -#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1328 -#define _LOAD_GLOBAL_r00 1329 -#define _LOAD_GLOBAL_BUILTINS_r01 1330 -#define _LOAD_GLOBAL_MODULE_r01 1331 -#define _LOAD_LOCALS_r01 1332 -#define _LOAD_LOCALS_r12 1333 -#define _LOAD_LOCALS_r23 1334 -#define _LOAD_NAME_r01 1335 -#define _LOAD_SMALL_INT_r01 1336 -#define _LOAD_SMALL_INT_r12 1337 -#define _LOAD_SMALL_INT_r23 1338 -#define _LOAD_SMALL_INT_0_r01 1339 -#define _LOAD_SMALL_INT_0_r12 1340 -#define _LOAD_SMALL_INT_0_r23 1341 -#define _LOAD_SMALL_INT_1_r01 1342 -#define _LOAD_SMALL_INT_1_r12 1343 -#define _LOAD_SMALL_INT_1_r23 1344 -#define _LOAD_SMALL_INT_2_r01 1345 -#define _LOAD_SMALL_INT_2_r12 1346 -#define _LOAD_SMALL_INT_2_r23 1347 -#define _LOAD_SMALL_INT_3_r01 1348 -#define _LOAD_SMALL_INT_3_r12 1349 -#define _LOAD_SMALL_INT_3_r23 1350 -#define _LOAD_SPECIAL_r00 1351 -#define _LOAD_SUPER_ATTR_ATTR_r31 1352 -#define _LOAD_SUPER_ATTR_METHOD_r32 1353 -#define _LOCK_OBJECT_r01 1354 -#define _LOCK_OBJECT_r11 1355 -#define _LOCK_OBJECT_r22 1356 -#define _LOCK_OBJECT_r33 1357 -#define _MAKE_CALLARGS_A_TUPLE_r33 1358 -#define _MAKE_CELL_r00 1359 -#define _MAKE_FUNCTION_r11 1360 -#define _MAKE_HEAP_SAFE_r01 1361 -#define _MAKE_HEAP_SAFE_r11 1362 -#define _MAKE_HEAP_SAFE_r22 1363 -#define _MAKE_HEAP_SAFE_r33 1364 -#define _MAKE_WARM_r00 1365 -#define _MAKE_WARM_r11 1366 -#define _MAKE_WARM_r22 1367 -#define _MAKE_WARM_r33 1368 -#define _MAP_ADD_r20 1369 -#define _MATCH_CLASS_r33 1370 -#define _MATCH_KEYS_r23 1371 -#define _MATCH_MAPPING_r02 1372 -#define _MATCH_MAPPING_r12 1373 -#define _MATCH_MAPPING_r23 1374 -#define _MATCH_SEQUENCE_r02 1375 -#define _MATCH_SEQUENCE_r12 1376 -#define _MATCH_SEQUENCE_r23 1377 -#define _MAYBE_EXPAND_METHOD_r00 1378 -#define _MAYBE_EXPAND_METHOD_KW_r11 1379 -#define _MONITOR_CALL_r00 1380 -#define _MONITOR_CALL_KW_r11 1381 -#define _MONITOR_JUMP_BACKWARD_r00 1382 -#define _MONITOR_JUMP_BACKWARD_r11 1383 -#define _MONITOR_JUMP_BACKWARD_r22 1384 -#define _MONITOR_JUMP_BACKWARD_r33 1385 -#define _MONITOR_RESUME_r00 1386 -#define _NOP_r00 1387 -#define _NOP_r11 1388 -#define _NOP_r22 1389 -#define _NOP_r33 1390 -#define _POP_CALL_r20 1391 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW_r21 1392 -#define _POP_CALL_ONE_r30 1393 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 1394 -#define _POP_CALL_TWO_r30 1395 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 1396 -#define _POP_EXCEPT_r10 1397 -#define _POP_ITER_r20 1398 -#define _POP_JUMP_IF_FALSE_r00 1399 -#define _POP_JUMP_IF_FALSE_r10 1400 -#define _POP_JUMP_IF_FALSE_r21 1401 -#define _POP_JUMP_IF_FALSE_r32 1402 -#define _POP_JUMP_IF_TRUE_r00 1403 -#define _POP_JUMP_IF_TRUE_r10 1404 -#define _POP_JUMP_IF_TRUE_r21 1405 -#define _POP_JUMP_IF_TRUE_r32 1406 -#define _POP_TOP_r10 1407 -#define _POP_TOP_FLOAT_r00 1408 -#define _POP_TOP_FLOAT_r10 1409 -#define _POP_TOP_FLOAT_r21 1410 -#define _POP_TOP_FLOAT_r32 1411 -#define _POP_TOP_INT_r00 1412 -#define _POP_TOP_INT_r10 1413 -#define _POP_TOP_INT_r21 1414 -#define _POP_TOP_INT_r32 1415 -#define _POP_TOP_LOAD_CONST_INLINE_r11 1416 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW_r11 1417 -#define _POP_TOP_NOP_r00 1418 -#define _POP_TOP_NOP_r10 1419 -#define _POP_TOP_NOP_r21 1420 -#define _POP_TOP_NOP_r32 1421 -#define _POP_TOP_UNICODE_r00 1422 -#define _POP_TOP_UNICODE_r10 1423 -#define _POP_TOP_UNICODE_r21 1424 -#define _POP_TOP_UNICODE_r32 1425 -#define _POP_TWO_r20 1426 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW_r21 1427 -#define _PUSH_EXC_INFO_r02 1428 -#define _PUSH_EXC_INFO_r12 1429 -#define _PUSH_EXC_INFO_r23 1430 -#define _PUSH_FRAME_r10 1431 -#define _PUSH_NULL_r01 1432 -#define _PUSH_NULL_r12 1433 -#define _PUSH_NULL_r23 1434 -#define _PUSH_NULL_CONDITIONAL_r00 1435 -#define _PY_FRAME_EX_r31 1436 -#define _PY_FRAME_GENERAL_r01 1437 -#define _PY_FRAME_KW_r11 1438 -#define _REPLACE_WITH_TRUE_r02 1439 -#define _REPLACE_WITH_TRUE_r12 1440 -#define _REPLACE_WITH_TRUE_r23 1441 -#define _RESUME_CHECK_r00 1442 -#define _RESUME_CHECK_r11 1443 -#define _RESUME_CHECK_r22 1444 -#define _RESUME_CHECK_r33 1445 -#define _RETURN_GENERATOR_r01 1446 -#define _RETURN_VALUE_r11 1447 -#define _SAVE_RETURN_OFFSET_r00 1448 -#define _SAVE_RETURN_OFFSET_r11 1449 -#define _SAVE_RETURN_OFFSET_r22 1450 -#define _SAVE_RETURN_OFFSET_r33 1451 -#define _SEND_r33 1452 -#define _SEND_GEN_FRAME_r33 1453 -#define _SETUP_ANNOTATIONS_r00 1454 -#define _SET_ADD_r10 1455 -#define _SET_FUNCTION_ATTRIBUTE_r01 1456 -#define _SET_FUNCTION_ATTRIBUTE_r11 1457 -#define _SET_FUNCTION_ATTRIBUTE_r21 1458 -#define _SET_FUNCTION_ATTRIBUTE_r32 1459 -#define _SET_IP_r00 1460 -#define _SET_IP_r11 1461 -#define _SET_IP_r22 1462 -#define _SET_IP_r33 1463 -#define _SET_UPDATE_r11 1464 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02 1465 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12 1466 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22 1467 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32 1468 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03 1469 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13 1470 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23 1471 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33 1472 -#define _SPILL_OR_RELOAD_r01 1473 -#define _SPILL_OR_RELOAD_r02 1474 -#define _SPILL_OR_RELOAD_r03 1475 -#define _SPILL_OR_RELOAD_r10 1476 -#define _SPILL_OR_RELOAD_r12 1477 -#define _SPILL_OR_RELOAD_r13 1478 -#define _SPILL_OR_RELOAD_r20 1479 -#define _SPILL_OR_RELOAD_r21 1480 -#define _SPILL_OR_RELOAD_r23 1481 -#define _SPILL_OR_RELOAD_r30 1482 -#define _SPILL_OR_RELOAD_r31 1483 -#define _SPILL_OR_RELOAD_r32 1484 -#define _START_EXECUTOR_r00 1485 -#define _STORE_ATTR_r20 1486 -#define _STORE_ATTR_INSTANCE_VALUE_r21 1487 -#define _STORE_ATTR_SLOT_r21 1488 -#define _STORE_ATTR_WITH_HINT_r21 1489 -#define _STORE_DEREF_r10 1490 -#define _STORE_FAST_LOAD_FAST_r11 1491 -#define _STORE_FAST_STORE_FAST_r20 1492 -#define _STORE_GLOBAL_r10 1493 -#define _STORE_NAME_r10 1494 -#define _STORE_SLICE_r30 1495 -#define _STORE_SUBSCR_r30 1496 -#define _STORE_SUBSCR_DICT_r31 1497 -#define _STORE_SUBSCR_DICT_KNOWN_HASH_r31 1498 -#define _STORE_SUBSCR_LIST_INT_r32 1499 -#define _SWAP_r11 1500 -#define _SWAP_2_r02 1501 -#define _SWAP_2_r12 1502 -#define _SWAP_2_r22 1503 -#define _SWAP_2_r33 1504 -#define _SWAP_3_r03 1505 -#define _SWAP_3_r13 1506 -#define _SWAP_3_r23 1507 -#define _SWAP_3_r33 1508 -#define _SWAP_FAST_r01 1509 -#define _SWAP_FAST_r11 1510 -#define _SWAP_FAST_r22 1511 -#define _SWAP_FAST_r33 1512 -#define _SWAP_FAST_0_r01 1513 -#define _SWAP_FAST_0_r11 1514 -#define _SWAP_FAST_0_r22 1515 -#define _SWAP_FAST_0_r33 1516 -#define _SWAP_FAST_1_r01 1517 -#define _SWAP_FAST_1_r11 1518 -#define _SWAP_FAST_1_r22 1519 -#define _SWAP_FAST_1_r33 1520 -#define _SWAP_FAST_2_r01 1521 -#define _SWAP_FAST_2_r11 1522 -#define _SWAP_FAST_2_r22 1523 -#define _SWAP_FAST_2_r33 1524 -#define _SWAP_FAST_3_r01 1525 -#define _SWAP_FAST_3_r11 1526 -#define _SWAP_FAST_3_r22 1527 -#define _SWAP_FAST_3_r33 1528 -#define _SWAP_FAST_4_r01 1529 -#define _SWAP_FAST_4_r11 1530 -#define _SWAP_FAST_4_r22 1531 -#define _SWAP_FAST_4_r33 1532 -#define _SWAP_FAST_5_r01 1533 -#define _SWAP_FAST_5_r11 1534 -#define _SWAP_FAST_5_r22 1535 -#define _SWAP_FAST_5_r33 1536 -#define _SWAP_FAST_6_r01 1537 -#define _SWAP_FAST_6_r11 1538 -#define _SWAP_FAST_6_r22 1539 -#define _SWAP_FAST_6_r33 1540 -#define _SWAP_FAST_7_r01 1541 -#define _SWAP_FAST_7_r11 1542 -#define _SWAP_FAST_7_r22 1543 -#define _SWAP_FAST_7_r33 1544 -#define _TIER2_RESUME_CHECK_r00 1545 -#define _TIER2_RESUME_CHECK_r11 1546 -#define _TIER2_RESUME_CHECK_r22 1547 -#define _TIER2_RESUME_CHECK_r33 1548 -#define _TO_BOOL_r11 1549 -#define _TO_BOOL_BOOL_r01 1550 -#define _TO_BOOL_BOOL_r11 1551 -#define _TO_BOOL_BOOL_r22 1552 -#define _TO_BOOL_BOOL_r33 1553 -#define _TO_BOOL_INT_r02 1554 -#define _TO_BOOL_INT_r12 1555 -#define _TO_BOOL_INT_r23 1556 -#define _TO_BOOL_LIST_r02 1557 -#define _TO_BOOL_LIST_r12 1558 -#define _TO_BOOL_LIST_r23 1559 -#define _TO_BOOL_NONE_r01 1560 -#define _TO_BOOL_NONE_r11 1561 -#define _TO_BOOL_NONE_r22 1562 -#define _TO_BOOL_NONE_r33 1563 -#define _TO_BOOL_STR_r02 1564 -#define _TO_BOOL_STR_r12 1565 -#define _TO_BOOL_STR_r23 1566 -#define _TRACE_RECORD_r00 1567 -#define _UNARY_INVERT_r12 1568 -#define _UNARY_NEGATIVE_r12 1569 -#define _UNARY_NEGATIVE_FLOAT_INPLACE_r02 1570 -#define _UNARY_NEGATIVE_FLOAT_INPLACE_r12 1571 -#define _UNARY_NEGATIVE_FLOAT_INPLACE_r23 1572 -#define _UNARY_NOT_r01 1573 -#define _UNARY_NOT_r11 1574 -#define _UNARY_NOT_r22 1575 -#define _UNARY_NOT_r33 1576 -#define _UNPACK_EX_r10 1577 -#define _UNPACK_SEQUENCE_r10 1578 -#define _UNPACK_SEQUENCE_LIST_r10 1579 -#define _UNPACK_SEQUENCE_TUPLE_r10 1580 -#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1581 -#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03 1582 -#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13 1583 -#define _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10 1584 -#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02 1585 -#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12 1586 -#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23 1587 -#define _WITH_EXCEPT_START_r33 1588 -#define _YIELD_VALUE_r11 1589 -#define MAX_UOP_REGS_ID 1589 +#define _INSERT_1_LOAD_CONST_INLINE_BORROW_r12 1154 +#define _INSERT_2_LOAD_CONST_INLINE_BORROW_r23 1155 +#define _INSERT_NULL_r10 1156 +#define _INSTRUMENTED_FOR_ITER_r23 1157 +#define _INSTRUMENTED_INSTRUCTION_r00 1158 +#define _INSTRUMENTED_JUMP_FORWARD_r00 1159 +#define _INSTRUMENTED_JUMP_FORWARD_r11 1160 +#define _INSTRUMENTED_JUMP_FORWARD_r22 1161 +#define _INSTRUMENTED_JUMP_FORWARD_r33 1162 +#define _INSTRUMENTED_LINE_r00 1163 +#define _INSTRUMENTED_NOT_TAKEN_r00 1164 +#define _INSTRUMENTED_NOT_TAKEN_r11 1165 +#define _INSTRUMENTED_NOT_TAKEN_r22 1166 +#define _INSTRUMENTED_NOT_TAKEN_r33 1167 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r00 1168 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 1169 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 1170 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 1171 +#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 1172 +#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 1173 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r00 1174 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 1175 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 1176 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 1177 +#define _IS_NONE_r11 1178 +#define _IS_OP_r03 1179 +#define _IS_OP_r13 1180 +#define _IS_OP_r23 1181 +#define _ITER_CHECK_LIST_r02 1182 +#define _ITER_CHECK_LIST_r12 1183 +#define _ITER_CHECK_LIST_r22 1184 +#define _ITER_CHECK_LIST_r33 1185 +#define _ITER_CHECK_RANGE_r02 1186 +#define _ITER_CHECK_RANGE_r12 1187 +#define _ITER_CHECK_RANGE_r22 1188 +#define _ITER_CHECK_RANGE_r33 1189 +#define _ITER_CHECK_TUPLE_r02 1190 +#define _ITER_CHECK_TUPLE_r12 1191 +#define _ITER_CHECK_TUPLE_r22 1192 +#define _ITER_CHECK_TUPLE_r33 1193 +#define _ITER_JUMP_LIST_r02 1194 +#define _ITER_JUMP_LIST_r12 1195 +#define _ITER_JUMP_LIST_r22 1196 +#define _ITER_JUMP_LIST_r33 1197 +#define _ITER_JUMP_RANGE_r02 1198 +#define _ITER_JUMP_RANGE_r12 1199 +#define _ITER_JUMP_RANGE_r22 1200 +#define _ITER_JUMP_RANGE_r33 1201 +#define _ITER_JUMP_TUPLE_r02 1202 +#define _ITER_JUMP_TUPLE_r12 1203 +#define _ITER_JUMP_TUPLE_r22 1204 +#define _ITER_JUMP_TUPLE_r33 1205 +#define _ITER_NEXT_LIST_r23 1206 +#define _ITER_NEXT_LIST_TIER_TWO_r23 1207 +#define _ITER_NEXT_RANGE_r03 1208 +#define _ITER_NEXT_RANGE_r13 1209 +#define _ITER_NEXT_RANGE_r23 1210 +#define _ITER_NEXT_TUPLE_r03 1211 +#define _ITER_NEXT_TUPLE_r13 1212 +#define _ITER_NEXT_TUPLE_r23 1213 +#define _JUMP_BACKWARD_NO_INTERRUPT_r00 1214 +#define _JUMP_BACKWARD_NO_INTERRUPT_r11 1215 +#define _JUMP_BACKWARD_NO_INTERRUPT_r22 1216 +#define _JUMP_BACKWARD_NO_INTERRUPT_r33 1217 +#define _JUMP_TO_TOP_r00 1218 +#define _LIST_APPEND_r10 1219 +#define _LIST_EXTEND_r11 1220 +#define _LOAD_ATTR_r10 1221 +#define _LOAD_ATTR_CLASS_r11 1222 +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_r11 1223 +#define _LOAD_ATTR_INSTANCE_VALUE_r02 1224 +#define _LOAD_ATTR_INSTANCE_VALUE_r12 1225 +#define _LOAD_ATTR_INSTANCE_VALUE_r23 1226 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r02 1227 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 1228 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 1229 +#define _LOAD_ATTR_METHOD_NO_DICT_r02 1230 +#define _LOAD_ATTR_METHOD_NO_DICT_r12 1231 +#define _LOAD_ATTR_METHOD_NO_DICT_r23 1232 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r02 1233 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 1234 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 1235 +#define _LOAD_ATTR_MODULE_r12 1236 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 1237 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 1238 +#define _LOAD_ATTR_PROPERTY_FRAME_r11 1239 +#define _LOAD_ATTR_SLOT_r02 1240 +#define _LOAD_ATTR_SLOT_r12 1241 +#define _LOAD_ATTR_SLOT_r23 1242 +#define _LOAD_ATTR_WITH_HINT_r12 1243 +#define _LOAD_BUILD_CLASS_r01 1244 +#define _LOAD_BYTECODE_r00 1245 +#define _LOAD_COMMON_CONSTANT_r01 1246 +#define _LOAD_COMMON_CONSTANT_r12 1247 +#define _LOAD_COMMON_CONSTANT_r23 1248 +#define _LOAD_CONST_r01 1249 +#define _LOAD_CONST_r12 1250 +#define _LOAD_CONST_r23 1251 +#define _LOAD_CONST_INLINE_r01 1252 +#define _LOAD_CONST_INLINE_r12 1253 +#define _LOAD_CONST_INLINE_r23 1254 +#define _LOAD_CONST_INLINE_BORROW_r01 1255 +#define _LOAD_DEREF_r01 1256 +#define _LOAD_FAST_r01 1257 +#define _LOAD_FAST_r12 1258 +#define _LOAD_FAST_r23 1259 +#define _LOAD_FAST_0_r01 1260 +#define _LOAD_FAST_0_r12 1261 +#define _LOAD_FAST_0_r23 1262 +#define _LOAD_FAST_1_r01 1263 +#define _LOAD_FAST_1_r12 1264 +#define _LOAD_FAST_1_r23 1265 +#define _LOAD_FAST_2_r01 1266 +#define _LOAD_FAST_2_r12 1267 +#define _LOAD_FAST_2_r23 1268 +#define _LOAD_FAST_3_r01 1269 +#define _LOAD_FAST_3_r12 1270 +#define _LOAD_FAST_3_r23 1271 +#define _LOAD_FAST_4_r01 1272 +#define _LOAD_FAST_4_r12 1273 +#define _LOAD_FAST_4_r23 1274 +#define _LOAD_FAST_5_r01 1275 +#define _LOAD_FAST_5_r12 1276 +#define _LOAD_FAST_5_r23 1277 +#define _LOAD_FAST_6_r01 1278 +#define _LOAD_FAST_6_r12 1279 +#define _LOAD_FAST_6_r23 1280 +#define _LOAD_FAST_7_r01 1281 +#define _LOAD_FAST_7_r12 1282 +#define _LOAD_FAST_7_r23 1283 +#define _LOAD_FAST_AND_CLEAR_r01 1284 +#define _LOAD_FAST_AND_CLEAR_r12 1285 +#define _LOAD_FAST_AND_CLEAR_r23 1286 +#define _LOAD_FAST_BORROW_r01 1287 +#define _LOAD_FAST_BORROW_r12 1288 +#define _LOAD_FAST_BORROW_r23 1289 +#define _LOAD_FAST_BORROW_0_r01 1290 +#define _LOAD_FAST_BORROW_0_r12 1291 +#define _LOAD_FAST_BORROW_0_r23 1292 +#define _LOAD_FAST_BORROW_1_r01 1293 +#define _LOAD_FAST_BORROW_1_r12 1294 +#define _LOAD_FAST_BORROW_1_r23 1295 +#define _LOAD_FAST_BORROW_2_r01 1296 +#define _LOAD_FAST_BORROW_2_r12 1297 +#define _LOAD_FAST_BORROW_2_r23 1298 +#define _LOAD_FAST_BORROW_3_r01 1299 +#define _LOAD_FAST_BORROW_3_r12 1300 +#define _LOAD_FAST_BORROW_3_r23 1301 +#define _LOAD_FAST_BORROW_4_r01 1302 +#define _LOAD_FAST_BORROW_4_r12 1303 +#define _LOAD_FAST_BORROW_4_r23 1304 +#define _LOAD_FAST_BORROW_5_r01 1305 +#define _LOAD_FAST_BORROW_5_r12 1306 +#define _LOAD_FAST_BORROW_5_r23 1307 +#define _LOAD_FAST_BORROW_6_r01 1308 +#define _LOAD_FAST_BORROW_6_r12 1309 +#define _LOAD_FAST_BORROW_6_r23 1310 +#define _LOAD_FAST_BORROW_7_r01 1311 +#define _LOAD_FAST_BORROW_7_r12 1312 +#define _LOAD_FAST_BORROW_7_r23 1313 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1314 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1315 +#define _LOAD_FAST_CHECK_r01 1316 +#define _LOAD_FAST_CHECK_r12 1317 +#define _LOAD_FAST_CHECK_r23 1318 +#define _LOAD_FAST_LOAD_FAST_r02 1319 +#define _LOAD_FAST_LOAD_FAST_r13 1320 +#define _LOAD_FROM_DICT_OR_DEREF_r11 1321 +#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1322 +#define _LOAD_GLOBAL_r00 1323 +#define _LOAD_GLOBAL_BUILTINS_r01 1324 +#define _LOAD_GLOBAL_MODULE_r01 1325 +#define _LOAD_LOCALS_r01 1326 +#define _LOAD_LOCALS_r12 1327 +#define _LOAD_LOCALS_r23 1328 +#define _LOAD_NAME_r01 1329 +#define _LOAD_SMALL_INT_r01 1330 +#define _LOAD_SMALL_INT_r12 1331 +#define _LOAD_SMALL_INT_r23 1332 +#define _LOAD_SMALL_INT_0_r01 1333 +#define _LOAD_SMALL_INT_0_r12 1334 +#define _LOAD_SMALL_INT_0_r23 1335 +#define _LOAD_SMALL_INT_1_r01 1336 +#define _LOAD_SMALL_INT_1_r12 1337 +#define _LOAD_SMALL_INT_1_r23 1338 +#define _LOAD_SMALL_INT_2_r01 1339 +#define _LOAD_SMALL_INT_2_r12 1340 +#define _LOAD_SMALL_INT_2_r23 1341 +#define _LOAD_SMALL_INT_3_r01 1342 +#define _LOAD_SMALL_INT_3_r12 1343 +#define _LOAD_SMALL_INT_3_r23 1344 +#define _LOAD_SPECIAL_r00 1345 +#define _LOAD_SUPER_ATTR_ATTR_r31 1346 +#define _LOAD_SUPER_ATTR_METHOD_r32 1347 +#define _LOCK_OBJECT_r01 1348 +#define _LOCK_OBJECT_r11 1349 +#define _LOCK_OBJECT_r22 1350 +#define _LOCK_OBJECT_r33 1351 +#define _MAKE_CALLARGS_A_TUPLE_r33 1352 +#define _MAKE_CELL_r00 1353 +#define _MAKE_FUNCTION_r11 1354 +#define _MAKE_HEAP_SAFE_r01 1355 +#define _MAKE_HEAP_SAFE_r11 1356 +#define _MAKE_HEAP_SAFE_r22 1357 +#define _MAKE_HEAP_SAFE_r33 1358 +#define _MAKE_WARM_r00 1359 +#define _MAKE_WARM_r11 1360 +#define _MAKE_WARM_r22 1361 +#define _MAKE_WARM_r33 1362 +#define _MAP_ADD_r20 1363 +#define _MATCH_CLASS_r33 1364 +#define _MATCH_KEYS_r23 1365 +#define _MATCH_MAPPING_r02 1366 +#define _MATCH_MAPPING_r12 1367 +#define _MATCH_MAPPING_r23 1368 +#define _MATCH_SEQUENCE_r02 1369 +#define _MATCH_SEQUENCE_r12 1370 +#define _MATCH_SEQUENCE_r23 1371 +#define _MAYBE_EXPAND_METHOD_r00 1372 +#define _MAYBE_EXPAND_METHOD_KW_r11 1373 +#define _MONITOR_CALL_r00 1374 +#define _MONITOR_CALL_KW_r11 1375 +#define _MONITOR_JUMP_BACKWARD_r00 1376 +#define _MONITOR_JUMP_BACKWARD_r11 1377 +#define _MONITOR_JUMP_BACKWARD_r22 1378 +#define _MONITOR_JUMP_BACKWARD_r33 1379 +#define _MONITOR_RESUME_r00 1380 +#define _NOP_r00 1381 +#define _NOP_r11 1382 +#define _NOP_r22 1383 +#define _NOP_r33 1384 +#define _POP_CALL_r20 1385 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW_r21 1386 +#define _POP_CALL_ONE_r30 1387 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 1388 +#define _POP_CALL_TWO_r30 1389 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 1390 +#define _POP_EXCEPT_r10 1391 +#define _POP_ITER_r20 1392 +#define _POP_JUMP_IF_FALSE_r00 1393 +#define _POP_JUMP_IF_FALSE_r10 1394 +#define _POP_JUMP_IF_FALSE_r21 1395 +#define _POP_JUMP_IF_FALSE_r32 1396 +#define _POP_JUMP_IF_TRUE_r00 1397 +#define _POP_JUMP_IF_TRUE_r10 1398 +#define _POP_JUMP_IF_TRUE_r21 1399 +#define _POP_JUMP_IF_TRUE_r32 1400 +#define _POP_TOP_r10 1401 +#define _POP_TOP_FLOAT_r00 1402 +#define _POP_TOP_FLOAT_r10 1403 +#define _POP_TOP_FLOAT_r21 1404 +#define _POP_TOP_FLOAT_r32 1405 +#define _POP_TOP_INT_r00 1406 +#define _POP_TOP_INT_r10 1407 +#define _POP_TOP_INT_r21 1408 +#define _POP_TOP_INT_r32 1409 +#define _POP_TOP_LOAD_CONST_INLINE_r11 1410 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW_r11 1411 +#define _POP_TOP_NOP_r00 1412 +#define _POP_TOP_NOP_r10 1413 +#define _POP_TOP_NOP_r21 1414 +#define _POP_TOP_NOP_r32 1415 +#define _POP_TOP_UNICODE_r00 1416 +#define _POP_TOP_UNICODE_r10 1417 +#define _POP_TOP_UNICODE_r21 1418 +#define _POP_TOP_UNICODE_r32 1419 +#define _POP_TWO_r20 1420 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW_r21 1421 +#define _PUSH_EXC_INFO_r02 1422 +#define _PUSH_EXC_INFO_r12 1423 +#define _PUSH_EXC_INFO_r23 1424 +#define _PUSH_FRAME_r10 1425 +#define _PUSH_NULL_r01 1426 +#define _PUSH_NULL_r12 1427 +#define _PUSH_NULL_r23 1428 +#define _PUSH_NULL_CONDITIONAL_r00 1429 +#define _PY_FRAME_EX_r31 1430 +#define _PY_FRAME_GENERAL_r01 1431 +#define _PY_FRAME_KW_r11 1432 +#define _REPLACE_WITH_TRUE_r02 1433 +#define _REPLACE_WITH_TRUE_r12 1434 +#define _REPLACE_WITH_TRUE_r23 1435 +#define _RESUME_CHECK_r00 1436 +#define _RESUME_CHECK_r11 1437 +#define _RESUME_CHECK_r22 1438 +#define _RESUME_CHECK_r33 1439 +#define _RETURN_GENERATOR_r01 1440 +#define _RETURN_VALUE_r11 1441 +#define _SAVE_RETURN_OFFSET_r00 1442 +#define _SAVE_RETURN_OFFSET_r11 1443 +#define _SAVE_RETURN_OFFSET_r22 1444 +#define _SAVE_RETURN_OFFSET_r33 1445 +#define _SEND_r33 1446 +#define _SEND_GEN_FRAME_r33 1447 +#define _SETUP_ANNOTATIONS_r00 1448 +#define _SET_ADD_r10 1449 +#define _SET_FUNCTION_ATTRIBUTE_r01 1450 +#define _SET_FUNCTION_ATTRIBUTE_r11 1451 +#define _SET_FUNCTION_ATTRIBUTE_r21 1452 +#define _SET_FUNCTION_ATTRIBUTE_r32 1453 +#define _SET_IP_r00 1454 +#define _SET_IP_r11 1455 +#define _SET_IP_r22 1456 +#define _SET_IP_r33 1457 +#define _SET_UPDATE_r11 1458 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32 1459 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33 1460 +#define _SPILL_OR_RELOAD_r01 1461 +#define _SPILL_OR_RELOAD_r02 1462 +#define _SPILL_OR_RELOAD_r03 1463 +#define _SPILL_OR_RELOAD_r10 1464 +#define _SPILL_OR_RELOAD_r12 1465 +#define _SPILL_OR_RELOAD_r13 1466 +#define _SPILL_OR_RELOAD_r20 1467 +#define _SPILL_OR_RELOAD_r21 1468 +#define _SPILL_OR_RELOAD_r23 1469 +#define _SPILL_OR_RELOAD_r30 1470 +#define _SPILL_OR_RELOAD_r31 1471 +#define _SPILL_OR_RELOAD_r32 1472 +#define _START_EXECUTOR_r00 1473 +#define _STORE_ATTR_r20 1474 +#define _STORE_ATTR_INSTANCE_VALUE_r21 1475 +#define _STORE_ATTR_SLOT_r21 1476 +#define _STORE_ATTR_WITH_HINT_r21 1477 +#define _STORE_DEREF_r10 1478 +#define _STORE_FAST_LOAD_FAST_r11 1479 +#define _STORE_FAST_STORE_FAST_r20 1480 +#define _STORE_GLOBAL_r10 1481 +#define _STORE_NAME_r10 1482 +#define _STORE_SLICE_r30 1483 +#define _STORE_SUBSCR_r30 1484 +#define _STORE_SUBSCR_DICT_r31 1485 +#define _STORE_SUBSCR_DICT_KNOWN_HASH_r31 1486 +#define _STORE_SUBSCR_LIST_INT_r32 1487 +#define _SWAP_r11 1488 +#define _SWAP_2_r02 1489 +#define _SWAP_2_r12 1490 +#define _SWAP_2_r22 1491 +#define _SWAP_2_r33 1492 +#define _SWAP_3_r03 1493 +#define _SWAP_3_r13 1494 +#define _SWAP_3_r23 1495 +#define _SWAP_3_r33 1496 +#define _SWAP_FAST_r01 1497 +#define _SWAP_FAST_r11 1498 +#define _SWAP_FAST_r22 1499 +#define _SWAP_FAST_r33 1500 +#define _SWAP_FAST_0_r01 1501 +#define _SWAP_FAST_0_r11 1502 +#define _SWAP_FAST_0_r22 1503 +#define _SWAP_FAST_0_r33 1504 +#define _SWAP_FAST_1_r01 1505 +#define _SWAP_FAST_1_r11 1506 +#define _SWAP_FAST_1_r22 1507 +#define _SWAP_FAST_1_r33 1508 +#define _SWAP_FAST_2_r01 1509 +#define _SWAP_FAST_2_r11 1510 +#define _SWAP_FAST_2_r22 1511 +#define _SWAP_FAST_2_r33 1512 +#define _SWAP_FAST_3_r01 1513 +#define _SWAP_FAST_3_r11 1514 +#define _SWAP_FAST_3_r22 1515 +#define _SWAP_FAST_3_r33 1516 +#define _SWAP_FAST_4_r01 1517 +#define _SWAP_FAST_4_r11 1518 +#define _SWAP_FAST_4_r22 1519 +#define _SWAP_FAST_4_r33 1520 +#define _SWAP_FAST_5_r01 1521 +#define _SWAP_FAST_5_r11 1522 +#define _SWAP_FAST_5_r22 1523 +#define _SWAP_FAST_5_r33 1524 +#define _SWAP_FAST_6_r01 1525 +#define _SWAP_FAST_6_r11 1526 +#define _SWAP_FAST_6_r22 1527 +#define _SWAP_FAST_6_r33 1528 +#define _SWAP_FAST_7_r01 1529 +#define _SWAP_FAST_7_r11 1530 +#define _SWAP_FAST_7_r22 1531 +#define _SWAP_FAST_7_r33 1532 +#define _TIER2_RESUME_CHECK_r00 1533 +#define _TIER2_RESUME_CHECK_r11 1534 +#define _TIER2_RESUME_CHECK_r22 1535 +#define _TIER2_RESUME_CHECK_r33 1536 +#define _TO_BOOL_r11 1537 +#define _TO_BOOL_BOOL_r01 1538 +#define _TO_BOOL_BOOL_r11 1539 +#define _TO_BOOL_BOOL_r22 1540 +#define _TO_BOOL_BOOL_r33 1541 +#define _TO_BOOL_INT_r02 1542 +#define _TO_BOOL_INT_r12 1543 +#define _TO_BOOL_INT_r23 1544 +#define _TO_BOOL_LIST_r02 1545 +#define _TO_BOOL_LIST_r12 1546 +#define _TO_BOOL_LIST_r23 1547 +#define _TO_BOOL_NONE_r01 1548 +#define _TO_BOOL_NONE_r11 1549 +#define _TO_BOOL_NONE_r22 1550 +#define _TO_BOOL_NONE_r33 1551 +#define _TO_BOOL_STR_r02 1552 +#define _TO_BOOL_STR_r12 1553 +#define _TO_BOOL_STR_r23 1554 +#define _TRACE_RECORD_r00 1555 +#define _UNARY_INVERT_r12 1556 +#define _UNARY_NEGATIVE_r12 1557 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r02 1558 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r12 1559 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r23 1560 +#define _UNARY_NOT_r01 1561 +#define _UNARY_NOT_r11 1562 +#define _UNARY_NOT_r22 1563 +#define _UNARY_NOT_r33 1564 +#define _UNPACK_EX_r10 1565 +#define _UNPACK_SEQUENCE_r10 1566 +#define _UNPACK_SEQUENCE_LIST_r10 1567 +#define _UNPACK_SEQUENCE_TUPLE_r10 1568 +#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1569 +#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03 1570 +#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13 1571 +#define _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10 1572 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02 1573 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12 1574 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23 1575 +#define _WITH_EXCEPT_START_r33 1576 +#define _YIELD_VALUE_r11 1577 +#define MAX_UOP_REGS_ID 1577 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index f3d334b85a24ba..2d455913071c31 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -380,7 +380,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, [_POP_TOP_LOAD_CONST_INLINE] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, + [_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_POP_CALL] = HAS_ESCAPES_FLAG, [_POP_CALL_ONE] = HAS_ESCAPES_FLAG, [_POP_CALL_TWO] = HAS_ESCAPES_FLAG, @@ -389,10 +389,10 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_POP_CALL_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, [_INSERT_1_LOAD_CONST_INLINE] = 0, - [_INSERT_1_LOAD_CONST_INLINE_BORROW] = 0, - [_INSERT_2_LOAD_CONST_INLINE_BORROW] = 0, - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW] = 0, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW] = 0, + [_INSERT_1_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, + [_INSERT_2_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, + [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, + [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, [_START_EXECUTOR] = HAS_DEOPT_FLAG, [_MAKE_WARM] = 0, @@ -3542,11 +3542,11 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { }, }, [_LOAD_CONST_INLINE_BORROW] = { - .best = { 0, 1, 2, 2 }, + .best = { 0, 0, 0, 0 }, .entries = { { 1, 0, _LOAD_CONST_INLINE_BORROW_r01 }, - { 2, 1, _LOAD_CONST_INLINE_BORROW_r12 }, - { 3, 2, _LOAD_CONST_INLINE_BORROW_r23 }, + { -1, -1, -1 }, + { -1, -1, -1 }, { -1, -1, -1 }, }, }, @@ -3623,38 +3623,38 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { }, }, [_INSERT_1_LOAD_CONST_INLINE_BORROW] = { - .best = { 0, 1, 2, 2 }, + .best = { 1, 1, 1, 1 }, .entries = { - { 2, 0, _INSERT_1_LOAD_CONST_INLINE_BORROW_r02 }, + { -1, -1, -1 }, { 2, 1, _INSERT_1_LOAD_CONST_INLINE_BORROW_r12 }, - { 3, 2, _INSERT_1_LOAD_CONST_INLINE_BORROW_r23 }, + { -1, -1, -1 }, { -1, -1, -1 }, }, }, [_INSERT_2_LOAD_CONST_INLINE_BORROW] = { - .best = { 0, 1, 2, 2 }, + .best = { 2, 2, 2, 2 }, .entries = { - { 3, 0, _INSERT_2_LOAD_CONST_INLINE_BORROW_r03 }, - { 3, 1, _INSERT_2_LOAD_CONST_INLINE_BORROW_r13 }, + { -1, -1, -1 }, + { -1, -1, -1 }, { 3, 2, _INSERT_2_LOAD_CONST_INLINE_BORROW_r23 }, { -1, -1, -1 }, }, }, [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW] = { - .best = { 0, 1, 2, 3 }, + .best = { 3, 3, 3, 3 }, .entries = { - { 2, 0, _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02 }, - { 2, 1, _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12 }, - { 2, 2, _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, { 2, 3, _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32 }, }, }, [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW] = { - .best = { 0, 1, 2, 3 }, + .best = { 3, 3, 3, 3 }, .entries = { - { 3, 0, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03 }, - { 3, 1, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13 }, - { 3, 2, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, { 3, 3, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33 }, }, }, @@ -4612,8 +4612,6 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_LOAD_CONST_INLINE_r23] = _LOAD_CONST_INLINE, [_POP_TOP_LOAD_CONST_INLINE_r11] = _POP_TOP_LOAD_CONST_INLINE, [_LOAD_CONST_INLINE_BORROW_r01] = _LOAD_CONST_INLINE_BORROW, - [_LOAD_CONST_INLINE_BORROW_r12] = _LOAD_CONST_INLINE_BORROW, - [_LOAD_CONST_INLINE_BORROW_r23] = _LOAD_CONST_INLINE_BORROW, [_POP_CALL_r20] = _POP_CALL, [_POP_CALL_ONE_r30] = _POP_CALL_ONE, [_POP_CALL_TWO_r30] = _POP_CALL_TWO, @@ -4624,19 +4622,9 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_INSERT_1_LOAD_CONST_INLINE_r02] = _INSERT_1_LOAD_CONST_INLINE, [_INSERT_1_LOAD_CONST_INLINE_r12] = _INSERT_1_LOAD_CONST_INLINE, [_INSERT_1_LOAD_CONST_INLINE_r23] = _INSERT_1_LOAD_CONST_INLINE, - [_INSERT_1_LOAD_CONST_INLINE_BORROW_r02] = _INSERT_1_LOAD_CONST_INLINE_BORROW, [_INSERT_1_LOAD_CONST_INLINE_BORROW_r12] = _INSERT_1_LOAD_CONST_INLINE_BORROW, - [_INSERT_1_LOAD_CONST_INLINE_BORROW_r23] = _INSERT_1_LOAD_CONST_INLINE_BORROW, - [_INSERT_2_LOAD_CONST_INLINE_BORROW_r03] = _INSERT_2_LOAD_CONST_INLINE_BORROW, - [_INSERT_2_LOAD_CONST_INLINE_BORROW_r13] = _INSERT_2_LOAD_CONST_INLINE_BORROW, [_INSERT_2_LOAD_CONST_INLINE_BORROW_r23] = _INSERT_2_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02] = _SHUFFLE_2_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12] = _SHUFFLE_2_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22] = _SHUFFLE_2_LOAD_CONST_INLINE_BORROW, [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32] = _SHUFFLE_2_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03] = _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13] = _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23] = _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33] = _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31] = _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, [_START_EXECUTOR_r00] = _START_EXECUTOR, @@ -5451,12 +5439,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_INSERT_1_LOAD_CONST_INLINE_r12] = "_INSERT_1_LOAD_CONST_INLINE_r12", [_INSERT_1_LOAD_CONST_INLINE_r23] = "_INSERT_1_LOAD_CONST_INLINE_r23", [_INSERT_1_LOAD_CONST_INLINE_BORROW] = "_INSERT_1_LOAD_CONST_INLINE_BORROW", - [_INSERT_1_LOAD_CONST_INLINE_BORROW_r02] = "_INSERT_1_LOAD_CONST_INLINE_BORROW_r02", [_INSERT_1_LOAD_CONST_INLINE_BORROW_r12] = "_INSERT_1_LOAD_CONST_INLINE_BORROW_r12", - [_INSERT_1_LOAD_CONST_INLINE_BORROW_r23] = "_INSERT_1_LOAD_CONST_INLINE_BORROW_r23", [_INSERT_2_LOAD_CONST_INLINE_BORROW] = "_INSERT_2_LOAD_CONST_INLINE_BORROW", - [_INSERT_2_LOAD_CONST_INLINE_BORROW_r03] = "_INSERT_2_LOAD_CONST_INLINE_BORROW_r03", - [_INSERT_2_LOAD_CONST_INLINE_BORROW_r13] = "_INSERT_2_LOAD_CONST_INLINE_BORROW_r13", [_INSERT_2_LOAD_CONST_INLINE_BORROW_r23] = "_INSERT_2_LOAD_CONST_INLINE_BORROW_r23", [_INSERT_NULL] = "_INSERT_NULL", [_INSERT_NULL_r10] = "_INSERT_NULL_r10", @@ -5547,8 +5531,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_LOAD_CONST_INLINE_r23] = "_LOAD_CONST_INLINE_r23", [_LOAD_CONST_INLINE_BORROW] = "_LOAD_CONST_INLINE_BORROW", [_LOAD_CONST_INLINE_BORROW_r01] = "_LOAD_CONST_INLINE_BORROW_r01", - [_LOAD_CONST_INLINE_BORROW_r12] = "_LOAD_CONST_INLINE_BORROW_r12", - [_LOAD_CONST_INLINE_BORROW_r23] = "_LOAD_CONST_INLINE_BORROW_r23", [_LOAD_DEREF] = "_LOAD_DEREF", [_LOAD_DEREF_r01] = "_LOAD_DEREF_r01", [_LOAD_FAST] = "_LOAD_FAST", @@ -5825,14 +5807,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_SET_UPDATE] = "_SET_UPDATE", [_SET_UPDATE_r11] = "_SET_UPDATE_r11", [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW] = "_SHUFFLE_2_LOAD_CONST_INLINE_BORROW", - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02] = "_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02", - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12] = "_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12", - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22] = "_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22", [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32] = "_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32", [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW", - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03", - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13", - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23", [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33", [_SPILL_OR_RELOAD] = "_SPILL_OR_RELOAD", [_SPILL_OR_RELOAD_r01] = "_SPILL_OR_RELOAD_r01", diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-04-19-35-32.gh-issue-145742.SU9RYL.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-04-19-35-32.gh-issue-145742.SU9RYL.rst new file mode 100644 index 00000000000000..0909ffe44bd331 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-04-19-35-32.gh-issue-145742.SU9RYL.rst @@ -0,0 +1,2 @@ +Optimize _LOAD_CONST_INLINE_BORROW by pre-tagging operands at trace +creation. Patch by Donghee Na. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 9e887ed84cfe0e..8bd8b0b44b43ee 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5879,7 +5879,7 @@ dummy_func( } tier2 pure op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - value = PyStackRef_FromPyObjectBorrow(ptr); + value = PyStackRef_FromPreTagged(ptr); } tier2 op(_POP_CALL, (callable, null --)) { @@ -5905,20 +5905,20 @@ dummy_func( tier2 op(_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { PyStackRef_CLOSE(pop); - value = PyStackRef_FromPyObjectBorrow(ptr); + value = PyStackRef_FromPreTagged(ptr); } tier2 op(_POP_TWO_LOAD_CONST_INLINE_BORROW, (ptr/4, pop1, pop2 -- value)) { PyStackRef_CLOSE(pop2); PyStackRef_CLOSE(pop1); - value = PyStackRef_FromPyObjectBorrow(ptr); + value = PyStackRef_FromPreTagged(ptr); } tier2 op(_POP_CALL_LOAD_CONST_INLINE_BORROW, (ptr/4, callable, null -- value)) { (void)null; // Silence compiler warnings about unused variables DEAD(null); PyStackRef_CLOSE(callable); - value = PyStackRef_FromPyObjectBorrow(ptr); + value = PyStackRef_FromPreTagged(ptr); } tier2 op(_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, (ptr/4, callable, null, pop -- value)) { @@ -5926,7 +5926,7 @@ dummy_func( (void)null; // Silence compiler warnings about unused variables DEAD(null); PyStackRef_CLOSE(callable); - value = PyStackRef_FromPyObjectBorrow(ptr); + value = PyStackRef_FromPreTagged(ptr); } tier2 op(_INSERT_1_LOAD_CONST_INLINE, (ptr/4, left -- res, l)) { @@ -5936,26 +5936,26 @@ dummy_func( } tier2 op(_INSERT_1_LOAD_CONST_INLINE_BORROW, (ptr/4, left -- res, l)) { - res = PyStackRef_FromPyObjectBorrow(ptr); + res = PyStackRef_FromPreTagged(ptr); l = left; INPUTS_DEAD(); } tier2 op(_INSERT_2_LOAD_CONST_INLINE_BORROW, (ptr/4, left, right -- res, l, r)) { - res = PyStackRef_FromPyObjectBorrow(ptr); + res = PyStackRef_FromPreTagged(ptr); l = left; r = right; INPUTS_DEAD(); } tier2 op(_SHUFFLE_2_LOAD_CONST_INLINE_BORROW, (ptr/4, callable, null, arg -- res, a)) { - res = PyStackRef_FromPyObjectBorrow(ptr); + res = PyStackRef_FromPreTagged(ptr); a = arg; INPUTS_DEAD(); } tier2 op(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW, (ptr/4, callable, null, arg -- res, a, c)) { - res = PyStackRef_FromPyObjectBorrow(ptr); + res = PyStackRef_FromPreTagged(ptr); a = arg; c = callable; INPUTS_DEAD(); @@ -5967,9 +5967,10 @@ dummy_func( (void)null; // Silence compiler warnings about unused variables DEAD(null); PyStackRef_CLOSE(callable); - value = PyStackRef_FromPyObjectBorrow(ptr); + value = PyStackRef_FromPreTagged(ptr); } + tier2 op(_START_EXECUTOR, (executor/4 --)) { #ifndef _Py_JIT assert(current_executor == (_PyExecutorObject*)executor); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 43a70315d81a86..7257438171d663 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -20987,43 +20987,17 @@ assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef value; PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - value = PyStackRef_FromPyObjectBorrow(ptr); + _PyFrame_SetStackPointer(frame, stack_pointer); + value = PyStackRef_FromPreTagged(ptr); + stack_pointer = _PyFrame_GetStackPointer(frame); _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _LOAD_CONST_INLINE_BORROW_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - value = PyStackRef_FromPyObjectBorrow(ptr); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_CONST_INLINE_BORROW_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - value = PyStackRef_FromPyObjectBorrow(ptr); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - case _POP_CALL_r20: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -21128,8 +21102,8 @@ PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); + value = PyStackRef_FromPreTagged(ptr); stack_pointer = _PyFrame_GetStackPointer(frame); - value = PyStackRef_FromPyObjectBorrow(ptr); _tos_cache0 = value; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -21159,8 +21133,8 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop1); + value = PyStackRef_FromPreTagged(ptr); stack_pointer = _PyFrame_GetStackPointer(frame); - value = PyStackRef_FromPyObjectBorrow(ptr); _tos_cache0 = value; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -21183,8 +21157,8 @@ (void)null; _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); + value = PyStackRef_FromPreTagged(ptr); stack_pointer = _PyFrame_GetStackPointer(frame); - value = PyStackRef_FromPyObjectBorrow(ptr); _tos_cache0 = value; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -21219,8 +21193,8 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); + value = PyStackRef_FromPreTagged(ptr); stack_pointer = _PyFrame_GetStackPointer(frame); - value = PyStackRef_FromPyObjectBorrow(ptr); _tos_cache0 = value; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -21286,25 +21260,6 @@ break; } - case _INSERT_1_LOAD_CONST_INLINE_BORROW_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - left = stack_pointer[-1]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - l = left; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - case _INSERT_1_LOAD_CONST_INLINE_BORROW_r12: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -21314,78 +21269,17 @@ _PyStackRef _stack_item_0 = _tos_cache0; left = _stack_item_0; PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - l = left; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _INSERT_1_LOAD_CONST_INLINE_BORROW_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - left = _stack_item_1; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - l = left; - _tos_cache2 = l; - _tos_cache1 = res; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _INSERT_2_LOAD_CONST_INLINE_BORROW_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; + stack_pointer[0] = left; + stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _INSERT_2_LOAD_CONST_INLINE_BORROW_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - right = _stack_item_0; - left = stack_pointer[-1]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); + _PyFrame_SetStackPointer(frame, stack_pointer); + res = PyStackRef_FromPreTagged(ptr); + stack_pointer = _PyFrame_GetStackPointer(frame); l = left; - r = right; - _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -21405,77 +21299,25 @@ right = _stack_item_1; left = _stack_item_0; PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + res = PyStackRef_FromPreTagged(ptr); + stack_pointer = _PyFrame_GetStackPointer(frame); l = left; r = right; _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef res; - _PyStackRef a; - arg = stack_pointer[-1]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef res; - _PyStackRef a; - _PyStackRef _stack_item_0 = _tos_cache0; - arg = _stack_item_0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef res; - _PyStackRef a; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - arg = _stack_item_1; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - case _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32: { CHECK_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -21487,90 +21329,25 @@ _PyStackRef _stack_item_2 = _tos_cache2; arg = _stack_item_2; PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = arg; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + res = PyStackRef_FromPreTagged(ptr); + stack_pointer = _PyFrame_GetStackPointer(frame); a = arg; _tos_cache1 = a; _tos_cache0 = res; + _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef a; - _PyStackRef c; - arg = stack_pointer[-1]; - callable = stack_pointer[-3]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - c = callable; - _tos_cache2 = c; - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef a; - _PyStackRef c; - _PyStackRef _stack_item_0 = _tos_cache0; - arg = _stack_item_0; - callable = stack_pointer[-2]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - c = callable; - _tos_cache2 = c; - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef a; - _PyStackRef c; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - arg = _stack_item_1; - callable = stack_pointer[-1]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - c = callable; - _tos_cache2 = c; - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33: { CHECK_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -21585,13 +21362,22 @@ arg = _stack_item_2; callable = _stack_item_0; PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); + stack_pointer[0] = callable; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = arg; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + res = PyStackRef_FromPreTagged(ptr); + stack_pointer = _PyFrame_GetStackPointer(frame); a = arg; c = callable; _tos_cache2 = c; _tos_cache1 = a; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } @@ -21629,8 +21415,8 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); + value = PyStackRef_FromPreTagged(ptr); stack_pointer = _PyFrame_GetStackPointer(frame); - value = PyStackRef_FromPyObjectBorrow(ptr); _tos_cache0 = value; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index ed1619df2a188b..5fbd835c631227 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -175,14 +175,15 @@ convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj, bool insert) if (res == NULL) { return NULL; } + bool borrow = _Py_IsImmortal(res); if (insert) { - if (_Py_IsImmortal(res)) { + if (borrow) { inst->opcode = _INSERT_1_LOAD_CONST_INLINE_BORROW; } else { inst->opcode = _INSERT_1_LOAD_CONST_INLINE; } } else { - if (_Py_IsImmortal(res)) { + if (borrow) { inst->opcode = _LOAD_CONST_INLINE_BORROW; } else { inst->opcode = _LOAD_CONST_INLINE; @@ -192,7 +193,7 @@ convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj, bool insert) assert(inst[1].oparg & 1); } } - inst->operand0 = (uint64_t)res; + inst->operand0 = borrow ? PyStackRef_TagBorrow(res) : (uint64_t)res; return res; } @@ -341,10 +342,10 @@ optimize_to_bool( if (truthiness >= 0) { PyObject *load = truthiness ? Py_True : Py_False; if (insert_mode) { - ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); + ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(load)); } else { ADD_OP(_POP_TOP, 0, 0); - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(load)); } *result_ptr = sym_new_const(ctx, load); return 1; @@ -401,11 +402,11 @@ lookup_attr(JitOptContext *ctx, _PyBloomFilter *dependencies, _PyUOpInstruction if (pop) { ADD_OP(_POP_TOP, 0, 0); ADD_OP(immortal ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE, - 0, (uintptr_t)lookup); + 0, immortal ? PyStackRef_TagBorrow(lookup) : (uintptr_t)lookup); } else { ADD_OP(immortal ? _INSERT_1_LOAD_CONST_INLINE_BORROW : _INSERT_1_LOAD_CONST_INLINE, - 0, (uintptr_t)lookup); + 0, immortal ? PyStackRef_TagBorrow(lookup) : (uintptr_t)lookup); } PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 3ba404b81ea043..68a4063e4e3697 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -730,14 +730,14 @@ dummy_func(void) { op(_LOAD_CONST, (-- value)) { PyCodeObject *co = get_current_code_object(ctx); PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(val)); value = PyJitRef_Borrow(sym_new_const(ctx, val)); } op(_LOAD_COMMON_CONSTANT, (-- value)) { assert(oparg < NUM_COMMON_CONSTANTS); PyObject *val = _PyInterpreterState_GET()->common_consts[oparg]; - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(val)); value = PyJitRef_Borrow(sym_new_const(ctx, val)); } @@ -745,7 +745,7 @@ dummy_func(void) { PyObject *val = PyLong_FromLong(oparg); assert(val); assert(_Py_IsImmortal(val)); - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(val)); value = PyJitRef_Borrow(sym_new_const(ctx, val)); } @@ -754,19 +754,19 @@ dummy_func(void) { } op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, PyStackRef_UntagBorrow(ptr))); } op(_POP_CALL_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused -- value)) { - value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, PyStackRef_UntagBorrow(ptr))); } op(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused, unused, unused -- value)) { - value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, PyStackRef_UntagBorrow(ptr))); } op(_SHUFFLE_2_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused, arg -- res, a)) { - res = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + res = PyJitRef_Borrow(sym_new_const(ctx, PyStackRef_UntagBorrow(ptr))); a = arg; } @@ -1215,7 +1215,7 @@ dummy_func(void) { if (type) { res = sym_new_const(ctx, type); ADD_OP(_SHUFFLE_2_LOAD_CONST_INLINE_BORROW, 0, - (uintptr_t)type); + PyStackRef_TagBorrow(type)); } else { res = sym_new_not_null(ctx); @@ -1252,7 +1252,7 @@ dummy_func(void) { out = Py_True; } sym_set_const(res, out); - ADD_OP(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); + ADD_OP(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(out)); } } @@ -1607,7 +1607,7 @@ dummy_func(void) { } op(_REPLACE_WITH_TRUE, (value -- res, v)) { - ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); + ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(Py_True)); res = sym_new_const(ctx, Py_True); v = value; } @@ -1892,7 +1892,7 @@ dummy_func(void) { goto error; } if (_Py_IsImmortal(temp)) { - ADD_OP(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); + ADD_OP(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(temp)); } res = sym_new_const(ctx, temp); Py_DECREF(temp); @@ -1913,7 +1913,7 @@ dummy_func(void) { goto error; } if (_Py_IsImmortal(temp)) { - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(temp)); } len = sym_new_const(ctx, temp); Py_DECREF(temp); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 2c8a0bff13ce35..74ff680fd4806d 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -78,7 +78,7 @@ JitOptRef value; PyCodeObject *co = get_current_code_object(ctx); PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(val)); value = PyJitRef_Borrow(sym_new_const(ctx, val)); CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; @@ -92,7 +92,7 @@ PyObject *val = PyLong_FromLong(oparg); assert(val); assert(_Py_IsImmortal(val)); - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(val)); value = PyJitRef_Borrow(sym_new_const(ctx, val)); CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; @@ -247,7 +247,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_1_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -316,7 +316,7 @@ if (_Py_IsImmortal(result)) { // Replace with _POP_TOP + _LOAD_CONST_INLINE_BORROW since we have one input and an immortal result ADD_OP(_POP_TOP, 0, 0); - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } stack_pointer[-1] = res; @@ -487,7 +487,7 @@ JitOptRef res; JitOptRef v; value = stack_pointer[-1]; - ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); + ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(Py_True)); res = sym_new_const(ctx, Py_True); v = value; CHECK_STACK_BOUNDS(1); @@ -526,7 +526,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_1_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -639,7 +639,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -709,7 +709,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -779,7 +779,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -1504,7 +1504,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -1811,7 +1811,7 @@ JitOptRef value; assert(oparg < NUM_COMMON_CONSTANTS); PyObject *val = _PyInterpreterState_GET()->common_consts[oparg]; - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(val)); value = PyJitRef_Borrow(sym_new_const(ctx, val)); CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; @@ -2642,7 +2642,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - ADD_OP(_POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_POP_TWO_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(-1); @@ -2713,7 +2713,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -2786,7 +2786,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -2848,7 +2848,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -2928,7 +2928,7 @@ PyObject *result = sym_get_const(ctx, b); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -3035,7 +3035,7 @@ PyObject *result = sym_get_const(ctx, b); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -3141,7 +3141,7 @@ goto error; } if (_Py_IsImmortal(temp)) { - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(temp)); } len = sym_new_const(ctx, temp); CHECK_STACK_BOUNDS(1); @@ -3739,7 +3739,7 @@ if (type) { res = sym_new_const(ctx, type); ADD_OP(_SHUFFLE_2_LOAD_CONST_INLINE_BORROW, 0, - (uintptr_t)type); + PyStackRef_TagBorrow(type)); } else { res = sym_new_not_null(ctx); @@ -4026,7 +4026,7 @@ goto error; } if (_Py_IsImmortal(temp)) { - ADD_OP(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); + ADD_OP(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(temp)); } res = sym_new_const(ctx, temp); CHECK_STACK_BOUNDS(-2); @@ -4072,7 +4072,7 @@ out = Py_True; } sym_set_const(res, out); - ADD_OP(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); + ADD_OP(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(out)); } CHECK_STACK_BOUNDS(-3); stack_pointer[-4] = res; @@ -4642,7 +4642,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, PyStackRef_TagBorrow(result)); } } CHECK_STACK_BOUNDS(1); @@ -4883,7 +4883,7 @@ case _LOAD_CONST_INLINE_BORROW: { JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, PyStackRef_UntagBorrow(ptr))); CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; @@ -4932,7 +4932,7 @@ case _POP_CALL_LOAD_CONST_INLINE_BORROW: { JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, PyStackRef_UntagBorrow(ptr))); CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = value; stack_pointer += -1; @@ -4998,7 +4998,7 @@ JitOptRef a; arg = stack_pointer[-1]; PyObject *ptr = (PyObject *)this_instr->operand0; - res = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + res = PyJitRef_Borrow(sym_new_const(ctx, PyStackRef_UntagBorrow(ptr))); a = arg; CHECK_STACK_BOUNDS(-1); stack_pointer[-3] = res; @@ -5024,7 +5024,7 @@ case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW: { JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, PyStackRef_UntagBorrow(ptr))); CHECK_STACK_BOUNDS(-3); stack_pointer[-4] = value; stack_pointer += -3; diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 6ba9c43ef1f0c3..030bf7c0bf16b8 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -766,7 +766,7 @@ def escaping_call_in_simple_stmt(stmt: SimpleStmt, result: dict[SimpleStmt, Esca continue #if not tkn.text.startswith(("Py", "_Py", "monitor")): # continue - if tkn.text.startswith(("sym_", "optimize_", "PyJitRef")): + if tkn.text.startswith(("sym_", "optimize_", "PyJitRef", "PyStackRef_Tag", "PyStackRef_Untag")): # Optimize functions continue if tkn.text.endswith("Check"): diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 495462b6bcd906..9ef0590803c3cb 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -241,20 +241,20 @@ def replace_opcode_if_evaluates_pure( # (a -- res), usually for unary ops 0: [("_POP_TOP", "0, 0"), ("_LOAD_CONST_INLINE_BORROW", - "0, (uintptr_t)result")], + "0, PyStackRef_TagBorrow(result)")], # (left -- res, left) # usually for unary ops with passthrough references 1: [("_INSERT_1_LOAD_CONST_INLINE_BORROW", - "0, (uintptr_t)result")], + "0, PyStackRef_TagBorrow(result)")], }, 2: { # (a, b -- res), usually for binary ops 0: [("_POP_TWO_LOAD_CONST_INLINE_BORROW", - "0, (uintptr_t)result")], + "0, PyStackRef_TagBorrow(result)")], # (left, right -- res, left, right) # usually for binary ops with passthrough references 2: [("_INSERT_2_LOAD_CONST_INLINE_BORROW", - "0, (uintptr_t)result")], + "0, PyStackRef_TagBorrow(result)")], }, }