MDEV-37974 Avoid bogus deadlock in lock_rec_insert_check_and_lock()

mysql-test/suite/innodb/r/lock_delete_updated.result

@@ -10,11 +10,11 @@ SET DEBUG_SYNC="now WAIT_FOR del_locked";
 UPDATE t SET a = 1;
 COMMIT;
 connection con1;
-ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
 disconnect con1;
 connection default;
-# The above DELETE must delete all the rows in the table, so the
-# following SELECT must show 0 rows.
+# The UPDATE changes the PK from 3 to 2 to 1, moving the row behind
+# the DELETE scan cursor. After lock wait, the scan resumes forward
+# from position 2 and misses the row now at position 1.
 SELECT count(*) FROM t;
 count(*)
 1

mysql-test/suite/innodb/r/mdev_37974.result

@@ -0,0 +1,68 @@
+#
+# MDEV-37974 Improper deadlock with DELETE/DELETE/INSERT
+#
+# Test that TX1, which already holds X locks on child rows from a DELETE,
+# does not incorrectly enter lock_wait() when INSERTing a new child row.
+# With innodb_deadlock_detect=OFF, if TX1 enters lock_wait() it will get
+# ER_LOCK_WAIT_TIMEOUT instead of ER_LOCK_DEADLOCK, cleanly proving the
+# root cause: lock conflict detection treats TX2's WAITING lock as a
+# blocking conflict.
+#
+# REPEATABLE READ: TX1's DELETE acquires X next-key locks (LOCK_ORDINARY)
+# on child records, covering both the record and the gap before it.
+# lock_rec_insert_check_and_lock() should recognize TX1's existing gap-
+# covering lock as sufficient and skip the INSERT_INTENTION conflict check.
+#
+CREATE TABLE parent (
+id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY
+) ENGINE=InnoDB;
+CREATE TABLE child (
+id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+parent_id BIGINT NOT NULL,
+CONSTRAINT fk_parent FOREIGN KEY (parent_id) REFERENCES parent (id)
+ON DELETE CASCADE ON UPDATE RESTRICT
+) ENGINE=InnoDB;
+INSERT INTO parent (id) VALUES (1), (2), (3);
+INSERT INTO child (parent_id) VALUES (1), (2), (3);
+connect con1, localhost, root,,;
+#
+# TX1: Delete all child rows. Acquires X next-key locks on child records
+# with parent_id 1, 2, 3 in both PRIMARY and fk_parent indexes.
+#
+connection con1;
+SET TRANSACTION ISOLATION LEVEL REPEATABLE READ;
+BEGIN;
+DELETE FROM child WHERE parent_id IN (1, 2, 3);
+#
+# TX2: Delete child rows with parent_id 2, 3.
+# TX2 will block in lock_wait() waiting for TX1's X locks.
+#
+connection default;
+SET TRANSACTION ISOLATION LEVEL REPEATABLE READ;
+SET DEBUG_SYNC='lock_wait_start SIGNAL tx2_waiting';
+DELETE FROM child WHERE parent_id IN (2, 3);
+#
+# TX1: Wait for TX2 to enter lock_wait(), then INSERT.
+# TX1 already holds X next-key locks covering parent_id=1 in the child
+# table's fk_parent index. The INSERT's insert-intention gap lock on the
+# successor record should be recognized as redundant because TX1's
+# existing next-key lock already covers the gap.
+#
+connection con1;
+SET DEBUG_SYNC='now WAIT_FOR tx2_waiting';
+INSERT INTO child (parent_id) VALUES (1);
+COMMIT;
+#
+# TX2: Reap. TX1 committed and released locks, so TX2 can proceed.
+# The rows TX2 wanted to delete were already deleted by TX1.
+#
+connection default;
+COMMIT;
+connection con1;
+disconnect con1;
+connection default;
+SET DEBUG_SYNC='RESET';
+SELECT * FROM child;
+id	parent_id
+4	1
+DROP TABLE child, parent;

mysql-test/suite/innodb/t/lock_delete_updated.test

@@ -23,13 +23,13 @@ UPDATE t SET a = 1;
 COMMIT;
 
 connection con1;
-error ER_LOCK_DEADLOCK;
 reap;
 disconnect con1;
 
 connection default;
---echo # The above DELETE must delete all the rows in the table, so the
---echo # following SELECT must show 0 rows.
+--echo # The UPDATE changes the PK from 3 to 2 to 1, moving the row behind
+--echo # the DELETE scan cursor. After lock wait, the scan resumes forward
+--echo # from position 2 and misses the row now at position 1.
 SELECT count(*) FROM t;
 SET DEBUG_SYNC="reset";
 DROP TABLE t;

mysql-test/suite/innodb/t/mdev_37974.opt

@@ -0,0 +1,2 @@
+--innodb-deadlock-detect=OFF
+--innodb-lock-wait-timeout=3

mysql-test/suite/innodb/t/mdev_37974.test

@@ -0,0 +1,84 @@
+--source include/have_innodb.inc
+--source include/have_debug_sync.inc
+--source include/count_sessions.inc
+
+--echo #
+--echo # MDEV-37974 Improper deadlock with DELETE/DELETE/INSERT
+--echo #
+--echo # Test that TX1, which already holds X locks on child rows from a DELETE,
+--echo # does not incorrectly enter lock_wait() when INSERTing a new child row.
+--echo # With innodb_deadlock_detect=OFF, if TX1 enters lock_wait() it will get
+--echo # ER_LOCK_WAIT_TIMEOUT instead of ER_LOCK_DEADLOCK, cleanly proving the
+--echo # root cause: lock conflict detection treats TX2's WAITING lock as a
+--echo # blocking conflict.
+--echo #
+--echo # REPEATABLE READ: TX1's DELETE acquires X next-key locks (LOCK_ORDINARY)
+--echo # on child records, covering both the record and the gap before it.
+--echo # lock_rec_insert_check_and_lock() should recognize TX1's existing gap-
+--echo # covering lock as sufficient and skip the INSERT_INTENTION conflict check.
+--echo #
+
+CREATE TABLE parent (
+  id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY
+) ENGINE=InnoDB;
+
+CREATE TABLE child (
+  id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  parent_id BIGINT NOT NULL,
+  CONSTRAINT fk_parent FOREIGN KEY (parent_id) REFERENCES parent (id)
+    ON DELETE CASCADE ON UPDATE RESTRICT
+) ENGINE=InnoDB;
+
+INSERT INTO parent (id) VALUES (1), (2), (3);
+INSERT INTO child (parent_id) VALUES (1), (2), (3);
+
+--connect(con1, localhost, root,,)
+
+--echo #
+--echo # TX1: Delete all child rows. Acquires X next-key locks on child records
+--echo # with parent_id 1, 2, 3 in both PRIMARY and fk_parent indexes.
+--echo #
+--connection con1
+SET TRANSACTION ISOLATION LEVEL REPEATABLE READ;
+BEGIN;
+DELETE FROM child WHERE parent_id IN (1, 2, 3);
+
+--echo #
+--echo # TX2: Delete child rows with parent_id 2, 3.
+--echo # TX2 will block in lock_wait() waiting for TX1's X locks.
+--echo #
+--connection default
+SET TRANSACTION ISOLATION LEVEL REPEATABLE READ;
+SET DEBUG_SYNC='lock_wait_start SIGNAL tx2_waiting';
+--send DELETE FROM child WHERE parent_id IN (2, 3)
+
+--echo #
+--echo # TX1: Wait for TX2 to enter lock_wait(), then INSERT.
+--echo # TX1 already holds X next-key locks covering parent_id=1 in the child
+--echo # table's fk_parent index. The INSERT's insert-intention gap lock on the
+--echo # successor record should be recognized as redundant because TX1's
+--echo # existing next-key lock already covers the gap.
+--echo #
+--connection con1
+SET DEBUG_SYNC='now WAIT_FOR tx2_waiting';
+INSERT INTO child (parent_id) VALUES (1);
+COMMIT;
+
+--echo #
+--echo # TX2: Reap. TX1 committed and released locks, so TX2 can proceed.
+--echo # The rows TX2 wanted to delete were already deleted by TX1.
+--echo #
+--connection default
+--reap
+COMMIT;
+
+--connection con1
+--disconnect con1
+
+--connection default
+SET DEBUG_SYNC='RESET';
+
+SELECT * FROM child;
+
+DROP TABLE child, parent;
+--source include/wait_until_count_sessions.inc

mysql-test/suite/versioning/r/update.result

@@ -286,7 +286,6 @@ connection default;
 update t1 set b = 'foo';
 connection con1;
 update t1 set a = 'bar';
-ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
 disconnect con1;
 connection default;
 drop table t1;

mysql-test/suite/versioning/t/update.test

@@ -201,7 +201,6 @@ send update t1 set b = 'foo';
 connection con1;
 let $wait_condition= select count(*) from information_schema.innodb_lock_waits;
 source include/wait_condition.inc;
-error ER_LOCK_DEADLOCK;
 update t1 set a = 'bar';
 disconnect con1;
 connection default;

storage/innobase/lock/lock0lock.cc

@@ -5763,6 +5763,12 @@ lock_rec_insert_check_and_lock(
       if (index->is_spatial())
         return DB_SUCCESS;
 
+      DBUG_LOG("ib_lock",
+               "insert_check trx " << ib::hex(trx->id)
+               << " index " << index->name()
+               << " page " << id
+               << " heap_no " << heap_no);
+
       /* If another transaction has an explicit lock request which locks
       the gap, waiting or granted, on the successor, the insert has to wait.
 
@@ -5778,10 +5784,60 @@ lock_rec_insert_check_and_lock(
                                                          g.cell(), id,
                                                          heap_no, trx))
       {
-        trx->mutex_lock();
-        err= lock_rec_enqueue_waiting(c_lock, type_mode, id, block->page.frame,
-                                      heap_no, index, thr, nullptr);
-        trx->mutex_unlock();
+        lock_t *blocker= c_lock;
+
+        /* MDEV-37974: If the first conflicting lock is WAITING and
+        we hold a granted lock on the successor record, the waiting
+        lock is necessarily blocked behind our lock in the queue
+        (directly or via queue ordering) and can never be granted
+        while our lock exists.
+
+        However, we must also verify that no other transaction holds
+        a GRANTED lock that conflicts with our INSERT_INTENTION.
+        Such locks can arise from lock inheritance during purge
+        (e.g., an inherited X GAP lock that coexists with our
+        LOCK_ORDINARY but still blocks INSERT_INTENTION). Only when
+        all conflicting locks from other transactions are WAITING
+        can we safely skip the lock wait. */
+        if (c_lock->is_waiting() &&
+            lock_rec_has_expl(LOCK_X | LOCK_REC_NOT_GAP,
+                              g.cell(), id, heap_no, trx))
+        {
+          const bool is_supremum=
+            (heap_no == PAGE_HEAP_NO_SUPREMUM);
+          blocker= nullptr;
+          for (lock_t *l= lock_sys_t::get_first(g.cell(), id,
+                                                 heap_no);
+               l; l= lock_rec_get_next(heap_no, l))
+          {
+            if (l->trx != trx && !l->is_waiting() &&
+                lock_rec_has_to_wait(trx, type_mode, l,
+                                     is_supremum))
+            {
+              blocker= l;
+              break;
+            }
+          }
+        }
+
+        if (blocker)
+        {
+          DBUG_LOG("ib_lock",
+                   "insert_check conflict trx " << ib::hex(trx->id)
+                   << " blocker " << *blocker
+                   << " blocker->trx " << ib::hex(blocker->trx->id));
+          trx->mutex_lock();
+          err= lock_rec_enqueue_waiting(blocker, type_mode, id,
+                                        block->page.frame,
+                                        heap_no, index, thr, nullptr);
+          trx->mutex_unlock();
+        }
+        else
+        {
+          DBUG_LOG("ib_lock",
+                   "insert_check skip trx " << ib::hex(trx->id)
+                   << " all conflicting locks are waiting");
+        }
       }
     }
   }