From b62b84adec167d7e5f5ee357f9ec681044269a07 Mon Sep 17 00:00:00 2001 From: Mohammad Tafzeel Shams Date: Fri, 17 Jul 2026 21:17:23 +0530 Subject: [PATCH] MDEV-39800: Assertion `!(mode & 2048U) || (mode & 512U) || is_supremum' failed ISSUE: Lock bypassing optimization allows an X-lock request to skip waiting locks when the requesting transaction already holds an S-lock on the same record. This optimization is designed for regular B-tree record locks that use heap-number-based conflict detection. Spatial index predicate locks use different semantics. They perform MBR (Minimum Bounding Rectangle) overlap checks for conflict detection and must not participate in bypass optimization. The assertion failure occurred because predicate insert intention locks is not considered inside lock bypass code, which assumed all insert intention locks must be either gap locks or on the supremum record (MDEV-34877). ut_ad(!(insert_intention) || (gap) || is_supremum) Additionally, predicate locks could incorrectly enable bypass_mode because the existing checks did not explicitly exclude LOCK_PREDICATE locks. FIX: - lock_t::is_predicate(): Add a helper to identify spatial index locks. - lock_t::can_be_bypassed(): Return false for predicate locks. - lock_rec_has_to_wait_in_queue(): Update the assertion to allow predicate insert intention locks. Also add a !is_predicate() check to bypass_mode calculation. - lock_rec_queue_validate_bypass(): Add an early return to skip bypass validation for predicate locks. --- .../suite/innodb_gis/r/rtree_debug.result | 30 +++++++++++++++++ .../suite/innodb_gis/t/rtree_debug.test | 33 +++++++++++++++++++ storage/innobase/include/lock0types.h | 9 ++++- storage/innobase/lock/lock0lock.cc | 12 +++++-- 4 files changed, 80 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/innodb_gis/r/rtree_debug.result b/mysql-test/suite/innodb_gis/r/rtree_debug.result index 7306a8b68f069..0804f6875d5fc 100644 --- a/mysql-test/suite/innodb_gis/r/rtree_debug.result +++ b/mysql-test/suite/innodb_gis/r/rtree_debug.result @@ -64,3 +64,33 @@ test.t1 check status OK truncate table t1; drop procedure insert_t1; drop table t1; +# +# MDEV-39800 Assertion `!(mode & 2048U) || (mode & 512U) || is_supremum' failed +# +CREATE TABLE t1 (pt POINT NOT NULL, SPATIAL INDEX(pt)) ENGINE=InnoDB; +INSERT INTO t1 VALUES (ST_GeomFromText('POINT(5 5)')); +connect con1,localhost,root,,test; +SET SESSION TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN; +SELECT COUNT(*) FROM t1 WHERE ST_Within(pt, ST_GeomFromText('POLYGON((0 0,100 0,100 100,0 100,0 0))')); +COUNT(*) +1 +connection default; +SET SESSION TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN; +SELECT COUNT(*) FROM t1 WHERE ST_Within(pt, ST_GeomFromText('POLYGON((0 0,100 0,100 100,0 100,0 0))')); +COUNT(*) +1 +connection con1; +INSERT INTO t1 VALUES (ST_GeomFromText('POINT(6 6)')); +connection default; +INSERT INTO t1 VALUES (ST_GeomFromText('POINT(7 7)')); +ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +connection con1; +COMMIT; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +connection default; +disconnect con1; +DROP TABLE t1; diff --git a/mysql-test/suite/innodb_gis/t/rtree_debug.test b/mysql-test/suite/innodb_gis/t/rtree_debug.test index 580f3cf48a16a..f88a84149c314 100644 --- a/mysql-test/suite/innodb_gis/t/rtree_debug.test +++ b/mysql-test/suite/innodb_gis/t/rtree_debug.test @@ -72,3 +72,36 @@ truncate table t1; # Clean up. drop procedure insert_t1; drop table t1; + +--echo # +--echo # MDEV-39800 Assertion `!(mode & 2048U) || (mode & 512U) || is_supremum' failed +--echo # +CREATE TABLE t1 (pt POINT NOT NULL, SPATIAL INDEX(pt)) ENGINE=InnoDB; +INSERT INTO t1 VALUES (ST_GeomFromText('POINT(5 5)')); + +--connect (con1,localhost,root,,test) +SET SESSION TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN; +SELECT COUNT(*) FROM t1 WHERE ST_Within(pt, ST_GeomFromText('POLYGON((0 0,100 0,100 100,0 100,0 0))')); + +--connection default +SET SESSION TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN; +SELECT COUNT(*) FROM t1 WHERE ST_Within(pt, ST_GeomFromText('POLYGON((0 0,100 0,100 100,0 100,0 0))')); + +--connection con1 +--send INSERT INTO t1 VALUES (ST_GeomFromText('POINT(6 6)')) + +--connection default +let $wait_condition= SELECT COUNT(*)=1 FROM information_schema.innodb_trx WHERE trx_state='LOCK WAIT'; +--source include/wait_condition.inc +--error ER_LOCK_DEADLOCK +INSERT INTO t1 VALUES (ST_GeomFromText('POINT(7 7)')); + +--connection con1 +--reap +COMMIT; +SELECT COUNT(*) FROM t1; +--connection default +--disconnect con1 +DROP TABLE t1; diff --git a/storage/innobase/include/lock0types.h b/storage/innobase/include/lock0types.h index da235fb06a041..828c37c65b889 100644 --- a/storage/innobase/include/lock0types.h +++ b/storage/innobase/include/lock0types.h @@ -225,6 +225,11 @@ struct ib_lock_t return(type_mode & LOCK_INSERT_INTENTION); } + bool is_predicate() const + { + return(type_mode & (LOCK_PREDICATE | LOCK_PRDT_PAGE)); + } + bool is_table() const { return type_mode & LOCK_TABLE; } enum lock_mode mode() const @@ -266,7 +271,9 @@ struct ib_lock_t condition here because there can be the following case: S1 X2(waits for S1) S3(waits for X2), bypassing X1 must not conflict with S3. */ - return has_s_lock_or_stronger && is_waiting_not_gap(); + /* Predicate locks do not participate in bypass mechanism. */ + return has_s_lock_or_stronger && is_waiting_not_gap() + && !is_predicate(); } /** Print the lock object into the given output stream. diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 768085c00603c..481dbf4627b3e 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -1256,6 +1256,7 @@ lock_rec_other_has_conflicting(unsigned mode, const hash_cell_t &cell, for (lock_t *lock= lock_sys_t::get_first(cell, id, heap_no); lock; lock= lock_rec_get_next(heap_no, lock)) { + ut_ad(!lock->is_predicate()); if (bypass_mode && lock_rec_can_be_bypassing(trx, lock)) { has_s_lock_or_stronger= true; @@ -1403,6 +1404,9 @@ in the lock bitmap. If wrong sequence is found, the function will crash with failed assertion. @param lock the lock which bitmap to be checked */ static void lock_rec_queue_validate_bypass(const lock_t *lock) { + /* Predicate locks do not participate in bypass mechanism. */ + if (lock->is_predicate()) + return; for (ulint i= 0; i < lock_rec_get_n_bits(lock); ++i) if (lock_rec_get_nth_bit(lock, i)) lock_rec_queue_validate_bypass(lock, i); @@ -1785,6 +1789,7 @@ static void lock_rec_add_to_queue(const conflicting_lock_info &c_lock_info, for (lock_t* lock = first_lock;;) { if (!lock_rec_get_nth_bit(lock, heap_no)) goto cont; + ut_ad(!lock->is_predicate()); ut_ad(!lock->is_insert_intention() || lock->is_gap() || is_supremum); if (bypass_mode && lock_rec_can_be_bypassing(trx, lock)) @@ -2067,9 +2072,10 @@ lock_rec_has_to_wait_in_queue(const hash_cell_t &cell, const lock_t *wait_lock) heap_no= lock_rec_find_set_bit(wait_lock); const bool is_supremum= (heap_no == PAGE_HEAP_NO_SUPREMUM); ut_ad(!(wait_lock->is_insert_intention()) || - (wait_lock->is_gap()) || is_supremum); - const bool bypass_mode= - !is_supremum && wait_lock->is_rec_exclusive_not_gap(); + (wait_lock->is_gap()) || (wait_lock->is_predicate()) || is_supremum); + /* Predicate locks do not participate in bypass mechanism. */ + const bool bypass_mode= !is_supremum + && wait_lock->is_rec_exclusive_not_gap() && !wait_lock->is_predicate(); bool has_s_lock_or_stronger= false; const lock_t *insert_after= nullptr; ut_d(const lock_t *bypassed= nullptr);