Optimize _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW

Author: tomasr8

Include/internal/pycore_uop_ids.h

@@ -1958,7 +1958,7 @@ def testfunc(n):
         self.assertNotIn("_CALL_ISINSTANCE", uops)
         self.assertNotIn("_GUARD_THIRD_NULL", uops)
         self.assertNotIn("_GUARD_CALLABLE_ISINSTANCE", uops)
-        self.assertIn("_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW", uops)
+        self.assertIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops)
 
     def test_call_list_append(self):
         def testfunc(n):
@@ -1991,7 +1991,7 @@ def testfunc(n):
         self.assertNotIn("_CALL_ISINSTANCE", uops)
         self.assertNotIn("_TO_BOOL_BOOL", uops)
         self.assertNotIn("_GUARD_IS_TRUE_POP", uops)
-        self.assertIn("_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW", uops)
+        self.assertIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops)
 
     def test_call_isinstance_is_false(self):
         def testfunc(n):
@@ -2009,7 +2009,7 @@ def testfunc(n):
         self.assertNotIn("_CALL_ISINSTANCE", uops)
         self.assertNotIn("_TO_BOOL_BOOL", uops)
         self.assertNotIn("_GUARD_IS_FALSE_POP", uops)
-        self.assertIn("_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW", uops)
+        self.assertIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops)
 
     def test_call_isinstance_subclass(self):
         def testfunc(n):
@@ -2027,7 +2027,7 @@ def testfunc(n):
         self.assertNotIn("_CALL_ISINSTANCE", uops)
         self.assertNotIn("_TO_BOOL_BOOL", uops)
         self.assertNotIn("_GUARD_IS_TRUE_POP", uops)
-        self.assertIn("_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW", uops)
+        self.assertIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops)
 
     def test_call_isinstance_unknown_object(self):
         def testfunc(n):

Include/internal/pycore_uop_metadata.h

@@ -5293,6 +5293,22 @@ dummy_func(
             value = PyStackRef_FromPyObjectBorrow(ptr);
         }
 
+        tier2 pure op(_POP_TWO, (unused, unused --)) {
+            // noop
+        }
+
+        tier2 pure op(_POP_THREE, (unused, unused, unused --)) {
+            // noop
+        }
+
+        tier2 pure op(_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, (ptr/4, callable, null, pop -- value)) {
+            PyStackRef_CLOSE(pop);
+            (void)null; // Silence compiler warnings about unused variables
+            DEAD(null);
+            PyStackRef_CLOSE(callable);
+            value = PyStackRef_FromPyObjectImmortal(ptr);
+        }
+
         tier2 pure op(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, (ptr/4, callable, null, pop1, pop2 -- value)) {
             PyStackRef_CLOSE(pop2);
             PyStackRef_CLOSE(pop1);

Lib/test/test_capi/test_opt.py

@@ -552,16 +552,26 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size)
                 }
                 break;
             case _POP_TOP:
+            case _POP_TWO:
+            case _POP_THREE:
             case _POP_TOP_LOAD_CONST_INLINE:
             case _POP_TOP_LOAD_CONST_INLINE_BORROW:
             case _POP_TWO_LOAD_CONST_INLINE_BORROW:
+            case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW:
+            case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW:
             optimize_pop_top_again:
             {
                 _PyUOpInstruction *last = &buffer[pc-1];
                 while (last->opcode == _NOP) {
                     last--;
                 }
                 switch (last->opcode) {
+                    case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW:
+                        last->opcode = _POP_THREE;
+                        break;
+                    case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW:
+                        last->opcode = _POP_TWO;
+                        break;
                     case _POP_TWO_LOAD_CONST_INLINE_BORROW:
                         last->opcode = _POP_TOP;
                         break;
@@ -579,17 +589,31 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size)
                         if (opcode == _POP_TOP) {
                             opcode = buffer[pc].opcode = _NOP;
                         }
+                        else if (opcode == _POP_TWO) {
+                            opcode = buffer[pc].opcode = _POP_TOP;
+                        }
+                        else if (opcode == _POP_THREE) {
+                            opcode = buffer[pc].opcode = _POP_TWO;
+                        }
                         else if (opcode == _POP_TOP_LOAD_CONST_INLINE) {
                             opcode = buffer[pc].opcode = _LOAD_CONST_INLINE;
                         }
                         else if (opcode == _POP_TOP_LOAD_CONST_INLINE_BORROW) {
                             opcode = buffer[pc].opcode = _LOAD_CONST_INLINE_BORROW;
                         }
-                        else {
-                            assert(opcode == _POP_TWO_LOAD_CONST_INLINE_BORROW);
+                        else if (opcode == _POP_TWO_LOAD_CONST_INLINE_BORROW) {
                             opcode = buffer[pc].opcode = _POP_TOP_LOAD_CONST_INLINE_BORROW;
                             goto optimize_pop_top_again;
                         }
+                        else if (opcode == _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW) {
+                            opcode = buffer[pc].opcode = _POP_TWO_LOAD_CONST_INLINE_BORROW;
+                            goto optimize_pop_top_again;
+                        }
+                        else {
+                            assert(opcode == _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW);
+                            opcode = buffer[pc].opcode = _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW;
+                            goto optimize_pop_top_again;
+                        }
                 }
                 _Py_FALLTHROUGH;
             }

Python/bytecodes.c

@@ -536,6 +536,10 @@ dummy_func(void) {
         value = sym_new_const(ctx, ptr);
     }
 
+    op(_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused, unused -- value)) {
+        value = sym_new_const(ctx, ptr);
+    }
+
     op(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused, unused, unused -- value)) {
         value = sym_new_const(ctx, ptr);
     }