diff --git a/mysql-test/suite/innodb_gis/r/rtree_debug.result b/mysql-test/suite/innodb_gis/r/rtree_debug.result index 7306a8b68f069..0804f6875d5fc 100644 --- a/mysql-test/suite/innodb_gis/r/rtree_debug.result +++ b/mysql-test/suite/innodb_gis/r/rtree_debug.result @@ -64,3 +64,33 @@ test.t1 check status OK truncate table t1; drop procedure insert_t1; drop table t1; +# +# MDEV-39800 Assertion `!(mode & 2048U) || (mode & 512U) || is_supremum' failed +# +CREATE TABLE t1 (pt POINT NOT NULL, SPATIAL INDEX(pt)) ENGINE=InnoDB; +INSERT INTO t1 VALUES (ST_GeomFromText('POINT(5 5)')); +connect con1,localhost,root,,test; +SET SESSION TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN; +SELECT COUNT(*) FROM t1 WHERE ST_Within(pt, ST_GeomFromText('POLYGON((0 0,100 0,100 100,0 100,0 0))')); +COUNT(*) +1 +connection default; +SET SESSION TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN; +SELECT COUNT(*) FROM t1 WHERE ST_Within(pt, ST_GeomFromText('POLYGON((0 0,100 0,100 100,0 100,0 0))')); +COUNT(*) +1 +connection con1; +INSERT INTO t1 VALUES (ST_GeomFromText('POINT(6 6)')); +connection default; +INSERT INTO t1 VALUES (ST_GeomFromText('POINT(7 7)')); +ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +connection con1; +COMMIT; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +connection default; +disconnect con1; +DROP TABLE t1; diff --git a/mysql-test/suite/innodb_gis/t/rtree_debug.test b/mysql-test/suite/innodb_gis/t/rtree_debug.test index 580f3cf48a16a..f88a84149c314 100644 --- a/mysql-test/suite/innodb_gis/t/rtree_debug.test +++ b/mysql-test/suite/innodb_gis/t/rtree_debug.test @@ -72,3 +72,36 @@ truncate table t1; # Clean up. drop procedure insert_t1; drop table t1; + +--echo # +--echo # MDEV-39800 Assertion `!(mode & 2048U) || (mode & 512U) || is_supremum' failed +--echo # +CREATE TABLE t1 (pt POINT NOT NULL, SPATIAL INDEX(pt)) ENGINE=InnoDB; +INSERT INTO t1 VALUES (ST_GeomFromText('POINT(5 5)')); + +--connect (con1,localhost,root,,test) +SET SESSION TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN; +SELECT COUNT(*) FROM t1 WHERE ST_Within(pt, ST_GeomFromText('POLYGON((0 0,100 0,100 100,0 100,0 0))')); + +--connection default +SET SESSION TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN; +SELECT COUNT(*) FROM t1 WHERE ST_Within(pt, ST_GeomFromText('POLYGON((0 0,100 0,100 100,0 100,0 0))')); + +--connection con1 +--send INSERT INTO t1 VALUES (ST_GeomFromText('POINT(6 6)')) + +--connection default +let $wait_condition= SELECT COUNT(*)=1 FROM information_schema.innodb_trx WHERE trx_state='LOCK WAIT'; +--source include/wait_condition.inc +--error ER_LOCK_DEADLOCK +INSERT INTO t1 VALUES (ST_GeomFromText('POINT(7 7)')); + +--connection con1 +--reap +COMMIT; +SELECT COUNT(*) FROM t1; +--connection default +--disconnect con1 +DROP TABLE t1; diff --git a/storage/innobase/include/lock0types.h b/storage/innobase/include/lock0types.h index da235fb06a041..828c37c65b889 100644 --- a/storage/innobase/include/lock0types.h +++ b/storage/innobase/include/lock0types.h @@ -225,6 +225,11 @@ struct ib_lock_t return(type_mode & LOCK_INSERT_INTENTION); } + bool is_predicate() const + { + return(type_mode & (LOCK_PREDICATE | LOCK_PRDT_PAGE)); + } + bool is_table() const { return type_mode & LOCK_TABLE; } enum lock_mode mode() const @@ -266,7 +271,9 @@ struct ib_lock_t condition here because there can be the following case: S1 X2(waits for S1) S3(waits for X2), bypassing X1 must not conflict with S3. */ - return has_s_lock_or_stronger && is_waiting_not_gap(); + /* Predicate locks do not participate in bypass mechanism. */ + return has_s_lock_or_stronger && is_waiting_not_gap() + && !is_predicate(); } /** Print the lock object into the given output stream. diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 768085c00603c..481dbf4627b3e 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -1256,6 +1256,7 @@ lock_rec_other_has_conflicting(unsigned mode, const hash_cell_t &cell, for (lock_t *lock= lock_sys_t::get_first(cell, id, heap_no); lock; lock= lock_rec_get_next(heap_no, lock)) { + ut_ad(!lock->is_predicate()); if (bypass_mode && lock_rec_can_be_bypassing(trx, lock)) { has_s_lock_or_stronger= true; @@ -1403,6 +1404,9 @@ in the lock bitmap. If wrong sequence is found, the function will crash with failed assertion. @param lock the lock which bitmap to be checked */ static void lock_rec_queue_validate_bypass(const lock_t *lock) { + /* Predicate locks do not participate in bypass mechanism. */ + if (lock->is_predicate()) + return; for (ulint i= 0; i < lock_rec_get_n_bits(lock); ++i) if (lock_rec_get_nth_bit(lock, i)) lock_rec_queue_validate_bypass(lock, i); @@ -1785,6 +1789,7 @@ static void lock_rec_add_to_queue(const conflicting_lock_info &c_lock_info, for (lock_t* lock = first_lock;;) { if (!lock_rec_get_nth_bit(lock, heap_no)) goto cont; + ut_ad(!lock->is_predicate()); ut_ad(!lock->is_insert_intention() || lock->is_gap() || is_supremum); if (bypass_mode && lock_rec_can_be_bypassing(trx, lock)) @@ -2067,9 +2072,10 @@ lock_rec_has_to_wait_in_queue(const hash_cell_t &cell, const lock_t *wait_lock) heap_no= lock_rec_find_set_bit(wait_lock); const bool is_supremum= (heap_no == PAGE_HEAP_NO_SUPREMUM); ut_ad(!(wait_lock->is_insert_intention()) || - (wait_lock->is_gap()) || is_supremum); - const bool bypass_mode= - !is_supremum && wait_lock->is_rec_exclusive_not_gap(); + (wait_lock->is_gap()) || (wait_lock->is_predicate()) || is_supremum); + /* Predicate locks do not participate in bypass mechanism. */ + const bool bypass_mode= !is_supremum + && wait_lock->is_rec_exclusive_not_gap() && !wait_lock->is_predicate(); bool has_s_lock_or_stronger= false; const lock_t *insert_after= nullptr; ut_d(const lock_t *bypassed= nullptr);