From bec0ba9a9791520a54ea88de60b0d4aa72993da9 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Wed, 10 Jun 2026 11:30:48 +0200 Subject: [PATCH 01/21] test: add failing tests for index-optimized queries mixing indexed and non-indexed conditions These tests assert the expected results of currentStateAsChanges for AND/OR where clauses that combine conditions on indexed fields with conditions that cannot be served by an index. They currently fail. Co-Authored-By: Claude Fable 5 --- packages/db/tests/collection-indexes.test.ts | 55 ++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/packages/db/tests/collection-indexes.test.ts b/packages/db/tests/collection-indexes.test.ts index a441a5520d..87f8d1a705 100644 --- a/packages/db/tests/collection-indexes.test.ts +++ b/packages/db/tests/collection-indexes.test.ts @@ -1179,6 +1179,61 @@ describe(`Collection Indexes`, () => { }) }) }) + + it(`should include rows matched by any OR condition when conditions mix indexed and non-indexed expressions`, () => { + // An OR query must return the union of rows matching each condition: + // eq(age, 25) matches Alice (age 25) + // gt(length(name), 6) matches Charlie (name length 7) + // `age` has an index while `length(name)` is a computed expression + // without one, but the chosen execution strategy must not change the + // result: both Alice and Charlie satisfy the OR and must be returned. + const result = collection.currentStateAsChanges({ + where: or( + eq(new PropRef([`age`]), 25), + gt(length(new PropRef([`name`])), 6), + ), + })! + + const names = result.map((r) => r.value.name).sort() + expect(names).toEqual([`Alice`, `Charlie`]) + }) + + it(`should only return rows matching every AND condition when conditions mix indexed and non-indexed expressions`, () => { + // An AND query must return only the rows matching all conditions: + // eq(status, 'active') matches Alice, Charlie and Eve + // gt(length(name), 6) matches only Charlie (name length 7) + // `status` has an index while `length(name)` is a computed expression + // without one, but every condition must still be enforced: only + // Charlie satisfies both. + const result = collection.currentStateAsChanges({ + where: and( + eq(new PropRef([`status`]), `active`), + gt(length(new PropRef([`name`])), 6), + ), + })! + + const names = result.map((r) => r.value.name).sort() + expect(names).toEqual([`Charlie`]) + }) + + it(`should enforce every AND condition when a range on one field is combined with conditions on other fields`, () => { + // An AND query that contains a compound range on one field plus a + // condition on another field must enforce all of them: + // gt(age, 24) AND lt(age, 36) matches Alice (25), Bob (30), + // Charlie (35) and Diana (28) + // eq(status, 'active') matches Alice, Charlie and Eve + // Only Alice and Charlie satisfy the full conjunction. + const result = collection.currentStateAsChanges({ + where: and( + gt(new PropRef([`age`]), 24), + lt(new PropRef([`age`]), 36), + eq(new PropRef([`status`]), `active`), + ), + })! + + const names = result.map((r) => r.value.name).sort() + expect(names).toEqual([`Alice`, `Charlie`]) + }) }) describe(`Index Usage Verification`, () => { From ac262057f543dd19bb91bd5ed331b0bf8cf680c9 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Wed, 10 Jun 2026 14:31:06 +0200 Subject: [PATCH 02/21] test: add failing tests for range query boundary handling Adds expected-behaviour tests for range conditions: - compound ranges sharing a boundary value must apply the strictest bound regardless of argument order, including for date values - one-sided compound ranges must return the matching rows - strict comparisons (gt) on date fields must exclude the boundary row Co-Authored-By: Claude Fable 5 --- packages/db/tests/collection-indexes.test.ts | 55 ++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/packages/db/tests/collection-indexes.test.ts b/packages/db/tests/collection-indexes.test.ts index 87f8d1a705..6faf07aa04 100644 --- a/packages/db/tests/collection-indexes.test.ts +++ b/packages/db/tests/collection-indexes.test.ts @@ -625,6 +625,19 @@ describe(`Collection Indexes`, () => { }) }) + it(`should exclude the boundary value from greater than queries on dates`, () => { + // gt must be strict for date fields: Bob was created exactly on + // 2023-01-02, so only rows created strictly later may be returned. + collection.createIndex((row) => row.createdAt) + + const result = collection.currentStateAsChanges({ + where: gt(new PropRef([`createdAt`]), new Date(`2023-01-02`)), + })! + + const names = result.map((r) => r.value.name).sort() + expect(names).toEqual([`Charlie`, `Diana`, `Eve`]) + }) + it(`should perform greater than or equal queries`, () => { withIndexTracking(collection, (tracker) => { const result = collection.currentStateAsChanges({ @@ -1216,6 +1229,48 @@ describe(`Collection Indexes`, () => { expect(names).toEqual([`Charlie`]) }) + it(`should apply the strictest lower bound when range conditions share the same value`, () => { + // gte(age, 25) AND gt(age, 25) reduces to age > 25: the strict + // comparison wins at the shared boundary, so Alice (age 25) must be + // excluded regardless of the order the conditions appear in. + const result = collection.currentStateAsChanges({ + where: and(gte(new PropRef([`age`]), 25), gt(new PropRef([`age`]), 25)), + })! + + const names = result.map((r) => r.value.name).sort() + expect(names).toEqual([`Bob`, `Charlie`, `Diana`]) + }) + + it(`should apply the strictest upper bound when range conditions share the same value`, () => { + // lte(age, 30) AND lt(age, 30) reduces to age < 30: the strict + // comparison wins at the shared boundary, so Bob (age 30) must be + // excluded. + const result = collection.currentStateAsChanges({ + where: and(lte(new PropRef([`age`]), 30), lt(new PropRef([`age`]), 30)), + })! + + const names = result.map((r) => r.value.name).sort() + expect(names).toEqual([`Alice`, `Diana`, `Eve`]) + }) + + it(`should apply the strictest bound for date ranges sharing the same value`, () => { + // Distinct Date instances representing the same point in time must be + // treated as equal values: gte(createdAt, jan2) AND gt(createdAt, jan2) + // reduces to createdAt > jan2, so Bob (created 2023-01-02) must be + // excluded. + collection.createIndex((row) => row.createdAt) + + const result = collection.currentStateAsChanges({ + where: and( + gte(new PropRef([`createdAt`]), new Date(`2023-01-02`)), + gt(new PropRef([`createdAt`]), new Date(`2023-01-02`)), + ), + })! + + const names = result.map((r) => r.value.name).sort() + expect(names).toEqual([`Charlie`, `Diana`, `Eve`]) + }) + it(`should enforce every AND condition when a range on one field is combined with conditions on other fields`, () => { // An AND query that contains a compound range on one field plus a // condition on another field must enforce all of them: From 0dac2f8a418f8e2bfc93c26af248dcbb66db81bb Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 18 Jun 2026 10:04:34 +0200 Subject: [PATCH 03/21] test: add failing test for compound range query with undefined bound A compound range condition where one bound is undefined (e.g. gt(score, undefined) AND lt(score, 90)) must match nothing, since a comparison against undefined is never true. The index-optimized path must agree with a full scan. This test currently fails. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/db/tests/collection-indexes.test.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packages/db/tests/collection-indexes.test.ts b/packages/db/tests/collection-indexes.test.ts index 6faf07aa04..5505b54dd7 100644 --- a/packages/db/tests/collection-indexes.test.ts +++ b/packages/db/tests/collection-indexes.test.ts @@ -1289,6 +1289,23 @@ describe(`Collection Indexes`, () => { const names = result.map((r) => r.value.name).sort() expect(names).toEqual([`Alice`, `Charlie`]) }) + + it(`should match a full scan when a range condition uses an undefined bound`, () => { + // A comparison against `undefined` matches no rows (a comparison with + // null/undefined is never true), so `gt(score, undefined)` excludes + // every row and the whole AND must return nothing. The index-optimized + // path must agree with a plain full scan and not leak rows. + collection.createIndex((row) => row.score) + + const result = collection.currentStateAsChanges({ + where: and( + gt(new PropRef([`score`]), undefined), + lt(new PropRef([`score`]), 90), + ), + })! + + expect(result).toEqual([]) + }) }) describe(`Index Usage Verification`, () => { From 029e759139d1b2cd584d12868ed1d40f22b55190 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Mon, 22 Jun 2026 10:11:21 +0200 Subject: [PATCH 04/21] test: add failing tests for nullish values in indexed eq/in/range queries A comparison against null/undefined is never true, but BTree indexes store and return rows with nullish indexed values (they sort as the smallest key). These tests assert that the index-optimized snapshot matches a full predicate scan for: - eq against undefined - IN with an undefined member - a range comparison over a field that has rows with undefined values - an upper-bounded compound range over such a field They currently fail. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/db/tests/collection-indexes.test.ts | 61 ++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/packages/db/tests/collection-indexes.test.ts b/packages/db/tests/collection-indexes.test.ts index 5505b54dd7..666e0a915c 100644 --- a/packages/db/tests/collection-indexes.test.ts +++ b/packages/db/tests/collection-indexes.test.ts @@ -1306,6 +1306,67 @@ describe(`Collection Indexes`, () => { expect(result).toEqual([]) }) + + it(`should not match rows with a missing value for an equality on undefined`, () => { + // An equality comparison against `undefined` is never true, so + // `eq(score, undefined)` must return no rows even though Eve has an + // undefined score. The index-optimized path must agree with a full + // predicate scan. + collection.createIndex((row) => row.score) + + const result = collection.currentStateAsChanges({ + where: eq(new PropRef([`score`]), undefined), + })! + + expect(result).toEqual([]) + }) + + it(`should ignore an undefined member when matching an IN list`, () => { + // A row only matches `IN` when its value equals one of the listed + // values; a comparison with `undefined` is never true. So + // `inArray(score, [undefined, 80])` must match only Bob (score 80) + // and must not match Eve (undefined score). + collection.createIndex((row) => row.score) + + const result = collection.currentStateAsChanges({ + where: inArray(new PropRef([`score`]), [undefined, 80]), + })! + + const names = result.map((r) => r.value.name).sort() + expect(names).toEqual([`Bob`]) + }) + + it(`should not match rows with a missing value for a range comparison`, () => { + // A range comparison against a row with an undefined value is never + // true, so `lt(score, 85)` must match only Bob (score 80) and must + // not match Eve (undefined score). + collection.createIndex((row) => row.score) + + const result = collection.currentStateAsChanges({ + where: lt(new PropRef([`score`]), 85), + })! + + const names = result.map((r) => r.value.name).sort() + expect(names).toEqual([`Bob`]) + }) + + it(`should not match rows with a missing value for an upper-bounded compound range`, () => { + // A compound range with only upper bounds (e.g. score <= 90) must not + // match a row with an undefined value, since a comparison against + // undefined is never true. Only Bob (80), Charlie (90) and Diana (85) + // satisfy `score <= 90`; Eve (undefined) must be excluded. + collection.createIndex((row) => row.score) + + const result = collection.currentStateAsChanges({ + where: and( + lte(new PropRef([`score`]), 90), + lte(new PropRef([`score`]), 95), + ), + })! + + const names = result.map((r) => r.value.name).sort() + expect(names).toEqual([`Bob`, `Charlie`, `Diana`]) + }) }) describe(`Index Usage Verification`, () => { From fc07196bb3fba30e9f95ce04d0c0c70838f55702 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Wed, 24 Jun 2026 11:53:38 +0200 Subject: [PATCH 05/21] test: add failing tests for locale string range and NaN index queries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more cases where an index-optimized snapshot must match a full predicate scan: - a string range predicate (e.g. name > 'z') must return a row whose value satisfies the JS relational comparison ('ö' > 'z'), even though a locale-collated index orders that value differently - eq and IN against NaN must not match a NaN-valued row, since NaN is never equal to itself They currently fail. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/db/tests/collection-indexes.test.ts | 99 ++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/packages/db/tests/collection-indexes.test.ts b/packages/db/tests/collection-indexes.test.ts index 666e0a915c..a63c3dfe86 100644 --- a/packages/db/tests/collection-indexes.test.ts +++ b/packages/db/tests/collection-indexes.test.ts @@ -1367,6 +1367,105 @@ describe(`Collection Indexes`, () => { const names = result.map((r) => r.value.name).sort() expect(names).toEqual([`Bob`, `Charlie`, `Diana`]) }) + + it(`should match a string range predicate using the same ordering as a full scan`, async () => { + // String comparisons in the WHERE evaluator use JS relational operators + // (code-point order), where `'ö' > 'z'` is true. A row named `ö` must + // therefore be returned by `name > 'z'`, even though a locale-collated + // index orders `ö` before `z`. The index-optimized result must agree + // with a full predicate scan. + const stringCollection = createCollection< + { id: string; name: string }, + string + >({ + getKey: (row) => row.id, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ type: `insert`, value: { id: `1`, name: `apple` } }) + write({ type: `insert`, value: { id: `2`, name: `ö` } }) + commit() + markReady() + }, + }, + }) + await stringCollection.stateWhenReady() + stringCollection.createIndex((row) => row.name) + + const result = stringCollection.currentStateAsChanges({ + where: gt(new PropRef([`name`]), `z`), + })! + + const names = result.map((r) => r.value.name).sort() + expect(names).toEqual([`ö`]) + }) + + it(`should not match a row with a NaN value for an equality on NaN`, async () => { + // NaN is never equal to itself, so `eq(score, NaN)` must return no rows + // even though the index stores and can return the NaN-valued row. + const nanCollection = createCollection< + { id: string; score: number }, + string + >({ + getKey: (row) => row.id, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ type: `insert`, value: { id: `1`, score: 5 } }) + write({ type: `insert`, value: { id: `2`, score: NaN } }) + commit() + markReady() + }, + }, + }) + await nanCollection.stateWhenReady() + nanCollection.createIndex((row) => row.score) + + const result = nanCollection.currentStateAsChanges({ + where: eq(new PropRef([`score`]), NaN), + })! + + expect(result).toEqual([]) + }) + + it(`should not match a row with a NaN value for an IN list containing NaN`, async () => { + // A row only matches `IN` when its value equals a listed value, and NaN + // is never equal to itself. So `inArray(score, [NaN, 5])` must match + // only the row with score 5 and never the NaN-valued row. + const nanCollection = createCollection< + { id: string; score: number }, + string + >({ + getKey: (row) => row.id, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ type: `insert`, value: { id: `1`, score: 5 } }) + write({ type: `insert`, value: { id: `2`, score: NaN } }) + commit() + markReady() + }, + }, + }) + await nanCollection.stateWhenReady() + nanCollection.createIndex((row) => row.score) + + const result = nanCollection.currentStateAsChanges({ + where: inArray(new PropRef([`score`]), [NaN, 5]), + })! + + const ids = result.map((r) => r.value.id).sort() + expect(ids).toEqual([`1`]) + }) }) describe(`Index Usage Verification`, () => { From 9d5c125bf81eacd3906d01555f2444e9a7ce00d9 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 25 Jun 2026 10:16:34 +0200 Subject: [PATCH 06/21] test: add failing tests for range predicates over non-orderable index domains Three more cases where an index-optimized range query must match a full predicate scan: - an array-valued field (the evaluator compares with standard relational operators, which differ from the index's recursive array ordering) - a field indexed with a custom comparator - a numeric field that also contains a NaN value They currently fail. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/db/tests/collection-indexes.test.ts | 104 +++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/packages/db/tests/collection-indexes.test.ts b/packages/db/tests/collection-indexes.test.ts index a63c3dfe86..ae1b45e844 100644 --- a/packages/db/tests/collection-indexes.test.ts +++ b/packages/db/tests/collection-indexes.test.ts @@ -1466,6 +1466,110 @@ describe(`Collection Indexes`, () => { const ids = result.map((r) => r.value.id).sort() expect(ids).toEqual([`1`]) }) + + it(`should return array-valued rows for a range predicate consistently with a full scan`, async () => { + // Range predicates are evaluated with standard relational comparison, + // under which `[2] > [10]` is true (arrays compare as their string + // form). An index on an array-valued field must return the same rows as + // a full scan and must not drop this match. + const arrayCollection = createCollection< + { id: string; value: Array }, + string + >({ + getKey: (row) => row.id, + startSync: true, + autoIndex: `off`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ type: `insert`, value: { id: `1`, value: [2] } }) + commit() + markReady() + }, + }, + }) + await arrayCollection.stateWhenReady() + arrayCollection.createIndex((row) => row.value) + + const result = arrayCollection.currentStateAsChanges({ + where: gt(new PropRef([`value`]), [10]), + })! + + const ids = result.map((r) => r.value.id).sort() + expect(ids).toEqual([`1`]) + }) + + it(`should return all matching rows for a range predicate on a custom-comparator index`, async () => { + // A range predicate must return every row that satisfies it regardless + // of the comparator the index was created with. With scores 5 and 20, + // `score > 10` matches only the row with score 20. + const customCollection = createCollection< + { id: string; score: number }, + string + >({ + getKey: (row) => row.id, + startSync: true, + autoIndex: `off`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ type: `insert`, value: { id: `low`, score: 5 } }) + write({ type: `insert`, value: { id: `high`, score: 20 } }) + commit() + markReady() + }, + }, + }) + await customCollection.stateWhenReady() + customCollection.createIndex((row) => row.score, { + options: { compareFn: (a: number, b: number) => b - a }, + }) + + const result = customCollection.currentStateAsChanges({ + where: gt(new PropRef([`score`]), 10), + })! + + const ids = result.map((r) => r.value.id).sort() + expect(ids).toEqual([`high`]) + }) + + it(`should return all matching rows for a range predicate when the field also contains NaN`, async () => { + // A range predicate must return every matching row even when other rows + // hold a NaN value for the field. With scores NaN, 1, 3, 5 and 7, + // `score > 2` matches the rows with scores 3, 5 and 7. + const nanCollection = createCollection< + { id: string; score: number }, + string + >({ + getKey: (row) => row.id, + startSync: true, + autoIndex: `off`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ type: `insert`, value: { id: `nan`, score: NaN } }) + write({ type: `insert`, value: { id: `one`, score: 1 } }) + write({ type: `insert`, value: { id: `three`, score: 3 } }) + write({ type: `insert`, value: { id: `five`, score: 5 } }) + write({ type: `insert`, value: { id: `seven`, score: 7 } }) + commit() + markReady() + }, + }, + }) + await nanCollection.stateWhenReady() + nanCollection.createIndex((row) => row.score) + + const result = nanCollection.currentStateAsChanges({ + where: gt(new PropRef([`score`]), 2), + })! + + const ids = result.map((r) => r.value.id).sort() + expect(ids).toEqual([`five`, `seven`, `three`]) + }) }) describe(`Index Usage Verification`, () => { From 13f7ead1e4eaeab5f3436dcd67a9daecc9ecb67f Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 25 Jun 2026 11:21:30 +0200 Subject: [PATCH 07/21] test: add failing tests for ordering values that have no natural order NaN and invalid Dates have no natural order. They should still get a consistent, well-defined position (alongside nulls) so that: - the comparator produces a stable total order; - ordering a collection by such a field is deterministic; - a range query on a field that contains such a value can still be served by the index rather than falling back to a full scan. These tests currently fail. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/db/tests/collection-indexes.test.ts | 43 +++++++++++++++++++ packages/db/tests/comparison.test.ts | 32 ++++++++++++++ .../db/tests/deterministic-ordering.test.ts | 33 ++++++++++++++ 3 files changed, 108 insertions(+) create mode 100644 packages/db/tests/comparison.test.ts diff --git a/packages/db/tests/collection-indexes.test.ts b/packages/db/tests/collection-indexes.test.ts index ae1b45e844..a9bb013efb 100644 --- a/packages/db/tests/collection-indexes.test.ts +++ b/packages/db/tests/collection-indexes.test.ts @@ -1570,6 +1570,49 @@ describe(`Collection Indexes`, () => { const ids = result.map((r) => r.value.id).sort() expect(ids).toEqual([`five`, `seven`, `three`]) }) + + it(`should use the index for a range query on a field that also contains NaN`, async () => { + // A NaN value has a well-defined sort position (with nulls), so a range + // query on the field can still be served by the index and does not need + // to fall back to a full scan. + const nanCollection = createCollection< + { id: string; score: number }, + string + >({ + getKey: (row) => row.id, + startSync: true, + autoIndex: `off`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ type: `insert`, value: { id: `nan`, score: NaN } }) + write({ type: `insert`, value: { id: `one`, score: 1 } }) + write({ type: `insert`, value: { id: `three`, score: 3 } }) + write({ type: `insert`, value: { id: `five`, score: 5 } }) + write({ type: `insert`, value: { id: `seven`, score: 7 } }) + commit() + markReady() + }, + }, + }) + await nanCollection.stateWhenReady() + nanCollection.createIndex((row) => row.score) + + withIndexTracking(nanCollection, (tracker) => { + const result = nanCollection.currentStateAsChanges({ + where: gt(new PropRef([`score`]), 2), + })! + + const ids = result.map((r) => r.value.id).sort() + expect(ids).toEqual([`five`, `seven`, `three`]) + + expectIndexUsage(tracker.stats, { + shouldUseIndex: true, + shouldUseFullScan: false, + }) + }) + }) }) describe(`Index Usage Verification`, () => { diff --git a/packages/db/tests/comparison.test.ts b/packages/db/tests/comparison.test.ts new file mode 100644 index 0000000000..a70c38981f --- /dev/null +++ b/packages/db/tests/comparison.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from 'vitest' +import { ascComparator, defaultComparator } from '../src/utils/comparison' +import { DEFAULT_COMPARE_OPTIONS } from '../src/utils' + +describe(`ascComparator with values that have no natural order`, () => { + const opts = DEFAULT_COMPARE_OPTIONS // nulls: `first` + + it(`should order NaN consistently relative to numbers`, () => { + // NaN has no natural order, but the comparator must still place it + // consistently (alongside nulls, which sort first by default) so the + // overall ordering stays well-defined. + expect(ascComparator(NaN, 5, opts)).toBeLessThan(0) + expect(ascComparator(5, NaN, opts)).toBeGreaterThan(0) + expect(ascComparator(NaN, NaN, opts)).toBe(0) + }) + + it(`should produce a stable total order when sorting numbers that include NaN`, () => { + const sorted = [3, NaN, 1, 5, NaN].sort((a, b) => defaultComparator(a, b)) + + // NaN values sort to the front (same end as nulls), the rest ascending + expect(sorted.slice(0, 2).every((v) => Number.isNaN(v))).toBe(true) + expect(sorted.slice(2)).toEqual([1, 3, 5]) + }) + + it(`should order an invalid Date consistently relative to valid Dates`, () => { + const invalid = new Date(`not a date`) + const valid = new Date(`2023-01-01`) + + expect(ascComparator(invalid, valid, opts)).toBeLessThan(0) + expect(ascComparator(valid, invalid, opts)).toBeGreaterThan(0) + }) +}) diff --git a/packages/db/tests/deterministic-ordering.test.ts b/packages/db/tests/deterministic-ordering.test.ts index 9ce9a326d6..2714688872 100644 --- a/packages/db/tests/deterministic-ordering.test.ts +++ b/packages/db/tests/deterministic-ordering.test.ts @@ -489,5 +489,38 @@ describe(`Deterministic Ordering`, () => { const keys = changes?.map((c) => c.key) expect(keys).toEqual([`a`, `b`, `c`]) }) + + it(`should place NaN values consistently when ordering`, () => { + type Item = { id: string; score: number } + + const options = mockSyncCollectionOptions({ + id: `test-collection-changes-nan`, + getKey: (item) => item.id, + initialData: [], + }) + + const collection = createCollection(options) + + options.utils.begin() + options.utils.write({ type: `insert`, value: { id: `a`, score: 5 } }) + options.utils.write({ type: `insert`, value: { id: `nan`, score: NaN } }) + options.utils.write({ type: `insert`, value: { id: `b`, score: 1 } }) + options.utils.write({ type: `insert`, value: { id: `c`, score: 3 } }) + options.utils.commit() + + const changes = collection.currentStateAsChanges({ + orderBy: [ + { + expression: new PropRef([`score`]), + compareOptions: { direction: `asc`, nulls: `first` }, + }, + ], + }) + + // NaN has no natural order; it sorts with nulls (first), then the + // numbers ascending. + const keys = changes?.map((c) => c.key) + expect(keys).toEqual([`nan`, `b`, `c`, `a`]) + }) }) }) From aa343b19b69c467c6db833b0066d352fc2bdc6cd Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Wed, 10 Jun 2026 14:31:51 +0200 Subject: [PATCH 08/21] fix: enforce all where conditions when index optimization is partial OR expressions now require every disjunct to be index-optimizable; otherwise the query falls back to a full scan, since rows matched only by a non-optimizable disjunct cannot be recovered from index lookups. AND expressions keep partial index optimization but the optimizer now reports whether the matching keys are exact. When they are a superset (some conjuncts could not use an index, or a compound range was combined with other conditions), currentStateAsChanges re-checks each candidate row against the full where expression. Co-Authored-By: Claude Fable 5 --- .../index-optimization-partial-and-or.md | 9 ++ packages/db/src/collection/change-events.ts | 9 +- packages/db/src/utils/index-optimization.ts | 128 +++++++++++++----- 3 files changed, 108 insertions(+), 38 deletions(-) create mode 100644 .changeset/index-optimization-partial-and-or.md diff --git a/.changeset/index-optimization-partial-and-or.md b/.changeset/index-optimization-partial-and-or.md new file mode 100644 index 0000000000..93296a50d7 --- /dev/null +++ b/.changeset/index-optimization-partial-and-or.md @@ -0,0 +1,9 @@ +--- +'@tanstack/db': patch +--- + +Fix incorrect results from index-optimized `where` clauses that combine indexed and non-indexed conditions. + +- `OR` expressions are now only served from indexes when every disjunct can use an index; otherwise the query falls back to a full scan. Previously, rows matched only by a non-indexed disjunct were missing from the result. +- `AND` expressions still use indexes for the conditions that have them, but the remaining conditions are now enforced by re-checking each candidate row against the full expression. Previously, non-indexed conditions were silently dropped, returning rows that did not match the query. +- Compound range conditions (e.g. `age > 5 AND age < 10`) combined with conditions on other fields no longer ignore those other conditions. diff --git a/packages/db/src/collection/change-events.ts b/packages/db/src/collection/change-events.ts index 6afac412ce..3f4977b7bd 100644 --- a/packages/db/src/collection/change-events.ts +++ b/packages/db/src/collection/change-events.ts @@ -138,11 +138,16 @@ export function currentStateAsChanges< ) if (optimizationResult.canOptimize) { - // Use index optimization + // Use index optimization. When the index lookup is inexact, the keys + // are a superset of the true result (some conditions could not be + // served by an index), so re-check each row against the full expression. + const filterFn = optimizationResult.isExact + ? undefined + : createFilterFunctionFromExpression(expression) const result: Array, TKey>> = [] for (const key of optimizationResult.matchingKeys) { const value = collection.get(key) - if (value !== undefined) { + if (value !== undefined && (filterFn?.(value) ?? true)) { result.push({ type: `insert`, key, diff --git a/packages/db/src/utils/index-optimization.ts b/packages/db/src/utils/index-optimization.ts index 81b111af56..7bb98ad470 100644 --- a/packages/db/src/utils/index-optimization.ts +++ b/packages/db/src/utils/index-optimization.ts @@ -29,6 +29,13 @@ import type { CollectionLike } from '../types.js' export interface OptimizationResult { canOptimize: boolean matchingKeys: Set + /** + * Whether `matchingKeys` is exactly the set of keys matching the expression. + * When `false`, the keys are a superset of the true result (some conditions + * could not be served by an index) and each row must be re-checked against + * the full expression before being included in the result. + */ + isExact: boolean } /** @@ -134,7 +141,7 @@ function optimizeQueryRecursive( } } - return { canOptimize: false, matchingKeys: new Set() } + return { canOptimize: false, matchingKeys: new Set(), isExact: false } } /** @@ -167,6 +174,14 @@ export function canOptimizeExpression< return false } +/** + * Result of compound range optimization, including which AND arguments + * were covered by the range query so the caller can process the rest. + */ +interface CompoundRangeResult extends OptimizationResult { + coveredArgIndices: Set +} + /** * Optimizes compound range queries on the same field * Example: WHERE age > 5 AND age < 10 @@ -177,9 +192,14 @@ function optimizeCompoundRangeQuery< >( expression: BasicExpression, collection: CollectionLike, -): OptimizationResult { +): CompoundRangeResult { if (expression.type !== `func` || expression.args.length < 2) { - return { canOptimize: false, matchingKeys: new Set() } + return { + canOptimize: false, + matchingKeys: new Set(), + isExact: false, + coveredArgIndices: new Set(), + } } // Group range operations by field @@ -188,11 +208,12 @@ function optimizeCompoundRangeQuery< Array<{ operation: `gt` | `gte` | `lt` | `lte` value: any + argIndex: number }> >() // Collect all range operations from AND arguments - for (const arg of expression.args) { + for (const [argIndex, arg] of expression.args.entries()) { if (arg.type === `func` && [`gt`, `gte`, `lt`, `lte`].includes(arg.name)) { const rangeOp = arg as any if (rangeOp.args.length === 2) { @@ -238,7 +259,7 @@ function optimizeCompoundRangeQuery< if (!fieldOperations.has(fieldKey)) { fieldOperations.set(fieldKey, []) } - fieldOperations.get(fieldKey)!.push({ operation, value }) + fieldOperations.get(fieldKey)!.push({ operation, value, argIndex }) } } } @@ -293,12 +314,22 @@ function optimizeCompoundRangeQuery< toInclusive, }) - return { canOptimize: true, matchingKeys } + return { + canOptimize: true, + matchingKeys, + isExact: true, + coveredArgIndices: new Set(operations.map((op) => op.argIndex)), + } } } } - return { canOptimize: false, matchingKeys: new Set() } + return { + canOptimize: false, + matchingKeys: new Set(), + isExact: false, + coveredArgIndices: new Set(), + } } /** @@ -312,7 +343,7 @@ function optimizeSimpleComparison< collection: CollectionLike, ): OptimizationResult { if (expression.type !== `func` || expression.args.length !== 2) { - return { canOptimize: false, matchingKeys: new Set() } + return { canOptimize: false, matchingKeys: new Set(), isExact: false } } const leftArg = expression.args[0]! @@ -362,15 +393,15 @@ function optimizeSimpleComparison< // Check if the index supports this operation if (!index.supports(indexOperation)) { - return { canOptimize: false, matchingKeys: new Set() } + return { canOptimize: false, matchingKeys: new Set(), isExact: false } } const matchingKeys = index.lookup(indexOperation, queryValue) - return { canOptimize: true, matchingKeys } + return { canOptimize: true, matchingKeys, isExact: true } } } - return { canOptimize: false, matchingKeys: new Set() } + return { canOptimize: false, matchingKeys: new Set(), isExact: false } } /** @@ -412,22 +443,38 @@ function optimizeAndExpression( collection: CollectionLike, ): OptimizationResult { if (expression.type !== `func` || expression.args.length < 2) { - return { canOptimize: false, matchingKeys: new Set() } + return { canOptimize: false, matchingKeys: new Set(), isExact: false } } // First, try to optimize compound range queries on the same field + // (e.g. age > 5 AND age < 10 becomes a single range query) const compoundRangeResult = optimizeCompoundRangeQuery(expression, collection) - if (compoundRangeResult.canOptimize) { - return compoundRangeResult - } + const coveredArgIndices = compoundRangeResult.canOptimize + ? compoundRangeResult.coveredArgIndices + : new Set() const results: Array> = [] + if (compoundRangeResult.canOptimize) { + results.push(compoundRangeResult) + } - // Try to optimize each part, keep the optimizable ones - for (const arg of expression.args) { + // Try to optimize the remaining conjuncts, keep the optimizable ones. + // Conjuncts that cannot use an index make the result inexact: the + // intersection is then a superset of the true result and must be + // re-filtered against the full expression by the caller. + let allConjunctsExact = true + for (const [argIndex, arg] of expression.args.entries()) { + if (coveredArgIndices.has(argIndex)) { + continue + } const result = optimizeQueryRecursive(arg, collection) if (result.canOptimize) { results.push(result) + if (!result.isExact) { + allConjunctsExact = false + } + } else { + allConjunctsExact = false } } @@ -435,10 +482,14 @@ function optimizeAndExpression( // Use intersectSets utility for AND logic const allMatchingSets = results.map((r) => r.matchingKeys) const intersectedKeys = intersectSets(allMatchingSets) - return { canOptimize: true, matchingKeys: intersectedKeys } + return { + canOptimize: true, + matchingKeys: intersectedKeys, + isExact: allConjunctsExact, + } } - return { canOptimize: false, matchingKeys: new Set() } + return { canOptimize: false, matchingKeys: new Set(), isExact: false } } /** @@ -464,27 +515,31 @@ function optimizeOrExpression( collection: CollectionLike, ): OptimizationResult { if (expression.type !== `func` || expression.args.length < 2) { - return { canOptimize: false, matchingKeys: new Set() } + return { canOptimize: false, matchingKeys: new Set(), isExact: false } } const results: Array> = [] - // Try to optimize each part, keep the optimizable ones + // Every disjunct must be optimizable: rows matched only by a disjunct + // that cannot use an index would be missing from the union, and no + // post-filtering can recover them. In that case fall back to a full scan. for (const arg of expression.args) { const result = optimizeQueryRecursive(arg, collection) - if (result.canOptimize) { - results.push(result) + if (!result.canOptimize) { + return { canOptimize: false, matchingKeys: new Set(), isExact: false } } + results.push(result) } - if (results.length > 0) { - // Use unionSets utility for OR logic - const allMatchingSets = results.map((r) => r.matchingKeys) - const unionedKeys = unionSets(allMatchingSets) - return { canOptimize: true, matchingKeys: unionedKeys } + // Use unionSets utility for OR logic + const allMatchingSets = results.map((r) => r.matchingKeys) + const unionedKeys = unionSets(allMatchingSets) + return { + canOptimize: true, + matchingKeys: unionedKeys, + // An inexact (superset) disjunct makes the union a superset as well + isExact: results.every((r) => r.isExact), } - - return { canOptimize: false, matchingKeys: new Set() } } /** @@ -498,8 +553,9 @@ function canOptimizeOrExpression< return false } - // If any argument can be optimized, we can gain some speedup - return expression.args.some((arg) => canOptimizeExpression(arg, collection)) + // Every disjunct must be optimizable, otherwise the union would miss + // rows matched only by the non-optimizable disjuncts + return expression.args.every((arg) => canOptimizeExpression(arg, collection)) } /** @@ -513,7 +569,7 @@ function optimizeInArrayExpression< collection: CollectionLike, ): OptimizationResult { if (expression.type !== `func` || expression.args.length !== 2) { - return { canOptimize: false, matchingKeys: new Set() } + return { canOptimize: false, matchingKeys: new Set(), isExact: false } } const fieldArg = expression.args[0]! @@ -532,7 +588,7 @@ function optimizeInArrayExpression< // Check if the index supports IN operation if (index.supports(`in`)) { const matchingKeys = index.lookup(`in`, values) - return { canOptimize: true, matchingKeys } + return { canOptimize: true, matchingKeys, isExact: true } } else if (index.supports(`eq`)) { // Fallback to multiple equality lookups const matchingKeys = new Set() @@ -542,12 +598,12 @@ function optimizeInArrayExpression< matchingKeys.add(key) } } - return { canOptimize: true, matchingKeys } + return { canOptimize: true, matchingKeys, isExact: true } } } } - return { canOptimize: false, matchingKeys: new Set() } + return { canOptimize: false, matchingKeys: new Set(), isExact: false } } /** From f18393ec23778cb4235672e7831925d819d4018a Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Wed, 10 Jun 2026 14:31:56 +0200 Subject: [PATCH 09/21] fix: apply strictest bound in compound range queries and fix related range edge cases Compound range conditions sharing a boundary value (gte(x,5) AND gt(x,5)) now keep the strict bound regardless of argument order. Bound values are compared with the same comparator the indexes use so dates and locale strings behave correctly. Two further issues surfaced by the regression tests: - One-sided compound ranges passed an explicit undefined bound to rangeQuery, which treats present-but-undefined as the undefined sentinel and returned an empty result. Bounds are now only passed when they exist. - BTreeIndex's exclusive lower bound check compared the normalized indexed value against the raw query value, so gt on date fields included the boundary row. It now compares against the normalized key. Co-Authored-By: Claude Fable 5 --- .../index-optimization-partial-and-or.md | 3 + packages/db/src/indexes/btree-index.ts | 5 +- packages/db/src/utils/index-optimization.ts | 58 ++++++++++++------- 3 files changed, 44 insertions(+), 22 deletions(-) diff --git a/.changeset/index-optimization-partial-and-or.md b/.changeset/index-optimization-partial-and-or.md index 93296a50d7..469975ad87 100644 --- a/.changeset/index-optimization-partial-and-or.md +++ b/.changeset/index-optimization-partial-and-or.md @@ -7,3 +7,6 @@ Fix incorrect results from index-optimized `where` clauses that combine indexed - `OR` expressions are now only served from indexes when every disjunct can use an index; otherwise the query falls back to a full scan. Previously, rows matched only by a non-indexed disjunct were missing from the result. - `AND` expressions still use indexes for the conditions that have them, but the remaining conditions are now enforced by re-checking each candidate row against the full expression. Previously, non-indexed conditions were silently dropped, returning rows that did not match the query. - Compound range conditions (e.g. `age > 5 AND age < 10`) combined with conditions on other fields no longer ignore those other conditions. +- Compound range conditions sharing the same boundary value (e.g. `age >= 5 AND age > 5`) now apply the strictest bound regardless of the order the conditions appear in, using the same value comparison semantics as the indexes (dates, locale strings, ...). +- Compound range conditions that only bound one side (e.g. `age > 5 AND age >= 8`) no longer return an empty result. +- Strict range comparisons (`gt`/`lt`) on BTree-indexed fields holding normalized values such as dates now correctly exclude the boundary value. diff --git a/packages/db/src/indexes/btree-index.ts b/packages/db/src/indexes/btree-index.ts index 17608950a7..1d04d928d5 100644 --- a/packages/db/src/indexes/btree-index.ts +++ b/packages/db/src/indexes/btree-index.ts @@ -247,7 +247,10 @@ export class BTreeIndex< toKey, toInclusive, (indexedValue, _) => { - if (!fromInclusive && this.compareFn(indexedValue, from) === 0) { + // Compare against the normalized key: indexed values are stored + // normalized (e.g. dates as timestamps), the raw `from` would + // never compare equal to them + if (!fromInclusive && this.compareFn(indexedValue, fromKey) === 0) { // the B+ tree `forRange` method does not support exclusive lower bounds // so we need to exclude it manually return diff --git a/packages/db/src/utils/index-optimization.ts b/packages/db/src/utils/index-optimization.ts index 7bb98ad470..7d88ddc1e8 100644 --- a/packages/db/src/utils/index-optimization.ts +++ b/packages/db/src/utils/index-optimization.ts @@ -18,6 +18,7 @@ import { DEFAULT_COMPARE_OPTIONS } from '../utils.js' import { ReverseIndex } from '../indexes/reverse-index.js' import { hasVirtualPropPath } from '../virtual-props.js' +import { makeComparator } from './comparison.js' import type { CompareOptions } from '../query/builder/types.js' import type { IndexInterface, IndexOperation } from '../indexes/base-index.js' import type { BasicExpression } from '../query/ir.js' @@ -272,7 +273,18 @@ function optimizeCompoundRangeQuery< const index = findIndexForField(collection, fieldPath) if (index && index.supports(`gt`) && index.supports(`lt`)) { - // Build range query options + // Compare values with the same semantics the index uses (dates, + // locale strings, ...), in ascending order since bounds are about + // value order regardless of the index direction + const compare = makeComparator({ + ...DEFAULT_COMPARE_OPTIONS, + ...collection.compareOptions, + direction: `asc`, + }) + + // Build range query options, keeping the strictest bound on each + // side: a larger lower bound (or smaller upper bound) wins, and at + // equal values the exclusive operation wins over the inclusive one let from: any = undefined let to: any = undefined let fromInclusive = true @@ -281,38 +293,42 @@ function optimizeCompoundRangeQuery< for (const { operation, value } of operations) { switch (operation) { case `gt`: - if (from === undefined || value > from) { + case `gte`: { + const cmp = from === undefined ? 1 : compare(value, from) + if (cmp > 0) { from = value + fromInclusive = operation === `gte` + } else if (cmp === 0 && operation === `gt`) { fromInclusive = false } break - case `gte`: - if (from === undefined || value > from) { - from = value - fromInclusive = true - } - break + } case `lt`: - if (to === undefined || value < to) { + case `lte`: { + const cmp = to === undefined ? -1 : compare(value, to) + if (cmp < 0) { to = value + toInclusive = operation === `lte` + } else if (cmp === 0 && operation === `lt`) { toInclusive = false } break - case `lte`: - if (to === undefined || value < to) { - to = value - toInclusive = true - } - break + } } } - const matchingKeys = (index as any).rangeQuery({ - from, - to, - fromInclusive, - toInclusive, - }) + // Only pass the bounds that exist: rangeQuery distinguishes an + // absent bound (open-ended) from an explicit undefined value + const rangeOptions: Record = {} + if (from !== undefined) { + rangeOptions.from = from + rangeOptions.fromInclusive = fromInclusive + } + if (to !== undefined) { + rangeOptions.to = to + rangeOptions.toInclusive = toInclusive + } + const matchingKeys = (index as any).rangeQuery(rangeOptions) return { canOptimize: true, From 4a4c30582c1a92f038523c8d56c127aa2caeec00 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 18 Jun 2026 10:05:24 +0200 Subject: [PATCH 10/21] test: add failing test for exclusive lower bound without a from bound rangeQuery with only an upper bound but fromInclusive: false must not drop the minimum key, as there is no lower bound to exclude against. This regression was introduced when the exclusive lower-bound check started comparing against the normalized fromKey (which defaults to minKey when no from bound is given). This test currently fails. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tests/btree-index-undefined-values.test.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/packages/db/tests/btree-index-undefined-values.test.ts b/packages/db/tests/btree-index-undefined-values.test.ts index 11e29690c6..1510c02e3c 100644 --- a/packages/db/tests/btree-index-undefined-values.test.ts +++ b/packages/db/tests/btree-index-undefined-values.test.ts @@ -248,6 +248,24 @@ describe(`BTreeIndex - undefined value handling`, () => { expect(withoutFrom.size).toBe(3) }) + it(`should not drop the minimum key when an upper-only range is exclusive on the (absent) lower bound`, () => { + // When no `from` bound is provided, `fromInclusive` must not cause the + // smallest key to be excluded: there is no lower bound to exclude + // against. Only an explicitly provided exclusive lower bound should + // drop its boundary value. + const index = createIndex(`value`) + index.add(`a`, { value: 1 }) + index.add(`b`, { value: 5 }) + index.add(`c`, { value: 10 }) + + const result = index.rangeQuery({ to: 10, fromInclusive: false }) + + expect(result.size).toBe(3) + expect(result).toContain(`a`) + expect(result).toContain(`b`) + expect(result).toContain(`c`) + }) + it(`should handle range query from undefined to undefined`, () => { const index = createIndex(`value`) index.add(`a`, { value: undefined }) From 1e17c2920494a6e3cea63c3c5a3320a20ac8423c Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 18 Jun 2026 10:12:54 +0200 Subject: [PATCH 11/21] fix: re-filter compound range queries that use a null/undefined bound A comparison against null/undefined is never true, but in an index those values sort as the smallest key, so an index range query cannot represent such a bound. Compound range optimization now tracks selected bounds with explicit hasFromBound/hasToBound flags (separate from the bound values) and marks the result inexact when any bound value is null/undefined, so the caller re-filters against the full expression. The inexactness now also propagates through the AND combiner, which previously ignored the compound range's exactness. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../index-optimization-partial-and-or.md | 1 + packages/db/src/utils/index-optimization.ts | 40 ++++++++++++++----- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/.changeset/index-optimization-partial-and-or.md b/.changeset/index-optimization-partial-and-or.md index 469975ad87..38ee6ab60e 100644 --- a/.changeset/index-optimization-partial-and-or.md +++ b/.changeset/index-optimization-partial-and-or.md @@ -10,3 +10,4 @@ Fix incorrect results from index-optimized `where` clauses that combine indexed - Compound range conditions sharing the same boundary value (e.g. `age >= 5 AND age > 5`) now apply the strictest bound regardless of the order the conditions appear in, using the same value comparison semantics as the indexes (dates, locale strings, ...). - Compound range conditions that only bound one side (e.g. `age > 5 AND age >= 8`) no longer return an empty result. - Strict range comparisons (`gt`/`lt`) on BTree-indexed fields holding normalized values such as dates now correctly exclude the boundary value. +- Compound range conditions with a `null`/`undefined` bound (e.g. `gt(score, undefined)`) now re-filter against the full expression instead of returning index-ordered rows, matching the semantics of a full scan (a comparison against `null`/`undefined` is never true). diff --git a/packages/db/src/utils/index-optimization.ts b/packages/db/src/utils/index-optimization.ts index 7d88ddc1e8..a5ebaddac8 100644 --- a/packages/db/src/utils/index-optimization.ts +++ b/packages/db/src/utils/index-optimization.ts @@ -284,19 +284,33 @@ function optimizeCompoundRangeQuery< // Build range query options, keeping the strictest bound on each // side: a larger lower bound (or smaller upper bound) wins, and at - // equal values the exclusive operation wins over the inclusive one + // equal values the exclusive operation wins over the inclusive one. + // `hasFromBound`/`hasToBound` track whether a bound was selected, + // separately from the bound value (which may legitimately be falsy). let from: any = undefined let to: any = undefined + let hasFromBound = false + let hasToBound = false let fromInclusive = true let toInclusive = true + // A comparison against null/undefined is never true, but in an index + // those values sort as the smallest key, so a range query cannot + // represent such a bound. Track it and force a re-filter instead of + // claiming the result is exact. + let hasNullBound = false for (const { operation, value } of operations) { + if (value == null) { + hasNullBound = true + continue + } switch (operation) { case `gt`: case `gte`: { - const cmp = from === undefined ? 1 : compare(value, from) + const cmp = hasFromBound ? compare(value, from) : 1 if (cmp > 0) { from = value + hasFromBound = true fromInclusive = operation === `gte` } else if (cmp === 0 && operation === `gt`) { fromInclusive = false @@ -305,9 +319,10 @@ function optimizeCompoundRangeQuery< } case `lt`: case `lte`: { - const cmp = to === undefined ? -1 : compare(value, to) + const cmp = hasToBound ? compare(value, to) : -1 if (cmp < 0) { to = value + hasToBound = true toInclusive = operation === `lte` } else if (cmp === 0 && operation === `lt`) { toInclusive = false @@ -317,14 +332,14 @@ function optimizeCompoundRangeQuery< } } - // Only pass the bounds that exist: rangeQuery distinguishes an - // absent bound (open-ended) from an explicit undefined value + // Only pass the bounds that were selected: rangeQuery distinguishes + // an absent bound (open-ended) from an explicitly provided one const rangeOptions: Record = {} - if (from !== undefined) { + if (hasFromBound) { rangeOptions.from = from rangeOptions.fromInclusive = fromInclusive } - if (to !== undefined) { + if (hasToBound) { rangeOptions.to = to rangeOptions.toInclusive = toInclusive } @@ -333,7 +348,9 @@ function optimizeCompoundRangeQuery< return { canOptimize: true, matchingKeys, - isExact: true, + // If a null/undefined bound was present, the range query result is + // a superset of the real matches and must be re-filtered + isExact: !hasNullBound, coveredArgIndices: new Set(operations.map((op) => op.argIndex)), } } @@ -477,8 +494,11 @@ function optimizeAndExpression( // Try to optimize the remaining conjuncts, keep the optimizable ones. // Conjuncts that cannot use an index make the result inexact: the // intersection is then a superset of the true result and must be - // re-filtered against the full expression by the caller. - let allConjunctsExact = true + // re-filtered against the full expression by the caller. The compound + // range result may itself be inexact (e.g. a null/undefined bound). + let allConjunctsExact = !compoundRangeResult.canOptimize + ? true + : compoundRangeResult.isExact for (const [argIndex, arg] of expression.args.entries()) { if (coveredArgIndices.has(argIndex)) { continue From 9a505ebf18be547ca2451596fcd2d840a616dd24 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 18 Jun 2026 10:13:04 +0200 Subject: [PATCH 12/21] fix: only exclude exclusive lower bound when a from bound is provided BTreeIndex.rangeQuery dropped the minimum key when called with fromInclusive: false but no from bound, because fromKey defaults to the minimum key and the exclusion check did not verify a lower bound was actually given. The exclusion is now guarded by hasFrom. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/db/src/indexes/btree-index.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/db/src/indexes/btree-index.ts b/packages/db/src/indexes/btree-index.ts index 1d04d928d5..b46c707102 100644 --- a/packages/db/src/indexes/btree-index.ts +++ b/packages/db/src/indexes/btree-index.ts @@ -247,10 +247,16 @@ export class BTreeIndex< toKey, toInclusive, (indexedValue, _) => { - // Compare against the normalized key: indexed values are stored - // normalized (e.g. dates as timestamps), the raw `from` would - // never compare equal to them - if (!fromInclusive && this.compareFn(indexedValue, fromKey) === 0) { + // Only exclude the boundary when an exclusive lower bound was + // actually provided. Without a `from` bound, `fromKey` defaults to + // the minimum key and must not be dropped. Compare against the + // normalized key since indexed values are stored normalized + // (e.g. dates as timestamps), so the raw `from` would never match. + if ( + hasFrom && + !fromInclusive && + this.compareFn(indexedValue, fromKey) === 0 + ) { // the B+ tree `forRange` method does not support exclusive lower bounds // so we need to exclude it manually return From 240f417c4018e6d341567c080b449f7d62073f96 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Mon, 22 Jun 2026 10:13:40 +0200 Subject: [PATCH 13/21] fix: re-filter index results that can include nullish-keyed rows BTree indexes store and return rows with a null/undefined indexed value (they sort as the smallest key), but a comparison against null/undefined is never true. The simple-comparison, IN, and compound-range optimizers now report such results as inexact so the caller re-checks candidates against the full expression: - eq/gt/gte: inexact when the query value is nullish (gt/gte with a non-null bound stay exact, since the bound excludes the bottom-sorted nullish rows) - lt/lte: conservatively inexact, as the open lower bound includes nullish-keyed rows - IN: inexact when any listed value is nullish - compound range: exact only when a non-null lower bound is present to exclude the nullish rows Co-Authored-By: Claude Opus 4.8 (1M context) --- .../index-optimization-partial-and-or.md | 1 + packages/db/src/utils/index-optimization.ts | 38 ++++++++++++++++--- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/.changeset/index-optimization-partial-and-or.md b/.changeset/index-optimization-partial-and-or.md index 38ee6ab60e..a67048c9d7 100644 --- a/.changeset/index-optimization-partial-and-or.md +++ b/.changeset/index-optimization-partial-and-or.md @@ -11,3 +11,4 @@ Fix incorrect results from index-optimized `where` clauses that combine indexed - Compound range conditions that only bound one side (e.g. `age > 5 AND age >= 8`) no longer return an empty result. - Strict range comparisons (`gt`/`lt`) on BTree-indexed fields holding normalized values such as dates now correctly exclude the boundary value. - Compound range conditions with a `null`/`undefined` bound (e.g. `gt(score, undefined)`) now re-filter against the full expression instead of returning index-ordered rows, matching the semantics of a full scan (a comparison against `null`/`undefined` is never true). +- Index-optimized `eq`, `IN`, and range queries on a field that has rows with `null`/`undefined` values no longer leak those rows into results. BTree indexes store and return such rows (they sort as the smallest key), but a comparison against `null`/`undefined` is never true, so these results are now re-filtered against the full expression to stay equivalent to a full scan. diff --git a/packages/db/src/utils/index-optimization.ts b/packages/db/src/utils/index-optimization.ts index a5ebaddac8..9f86983314 100644 --- a/packages/db/src/utils/index-optimization.ts +++ b/packages/db/src/utils/index-optimization.ts @@ -348,9 +348,13 @@ function optimizeCompoundRangeQuery< return { canOptimize: true, matchingKeys, - // If a null/undefined bound was present, the range query result is - // a superset of the real matches and must be re-filtered - isExact: !hasNullBound, + // The range result is exact only when it cannot include rows with a + // nullish indexed value (which a comparison would reject but the + // index returns, as they sort as the smallest key). That requires a + // non-nullish lower bound to exclude them: without `hasFromBound` + // the range is open at the bottom and captures those rows, and a + // nullish bound value (`hasNullBound`) can never bound them out. + isExact: hasFromBound && !hasNullBound, coveredArgIndices: new Set(operations.map((op) => op.argIndex)), } } @@ -430,7 +434,23 @@ function optimizeSimpleComparison< } const matchingKeys = index.lookup(indexOperation, queryValue) - return { canOptimize: true, matchingKeys, isExact: true } + + // A comparison against null/undefined is never true, but BTree indexes + // store and return rows with a nullish indexed value (they sort as the + // smallest key). Determine whether the index result is exact or a + // superset that the caller must re-filter: + // - eq/gt/gte: a nullish query value matches nothing, while the index + // would still return nullish-keyed rows -> inexact when nullish. A + // non-nullish lower bound (gt/gte) excludes the bottom-sorted nullish + // rows, so those stay exact. + // - lt/lte: the open lower bound always includes nullish-keyed rows, + // so the result is conservatively inexact. + const isExact = + operation === `lt` || operation === `lte` + ? false + : queryValue != null + + return { canOptimize: true, matchingKeys, isExact } } } @@ -620,11 +640,17 @@ function optimizeInArrayExpression< const values = (arrayArg as any).value const index = findIndexForField(collection, fieldPath) + // A nullish member can never be matched by `IN` (a comparison against + // null/undefined is never true), but the index would still return rows + // with a nullish indexed value. When the list contains a nullish member + // the result is a superset that the caller must re-filter. + const isExact = !values.some((value: any) => value == null) + if (index) { // Check if the index supports IN operation if (index.supports(`in`)) { const matchingKeys = index.lookup(`in`, values) - return { canOptimize: true, matchingKeys, isExact: true } + return { canOptimize: true, matchingKeys, isExact } } else if (index.supports(`eq`)) { // Fallback to multiple equality lookups const matchingKeys = new Set() @@ -634,7 +660,7 @@ function optimizeInArrayExpression< matchingKeys.add(key) } } - return { canOptimize: true, matchingKeys, isExact: true } + return { canOptimize: true, matchingKeys, isExact } } } } From f8084d8e4d0796ca47dca5063e8586f828e172c5 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Mon, 22 Jun 2026 08:14:50 +0000 Subject: [PATCH 14/21] ci: apply automated fixes --- packages/db/src/utils/index-optimization.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/db/src/utils/index-optimization.ts b/packages/db/src/utils/index-optimization.ts index 9f86983314..103cd63557 100644 --- a/packages/db/src/utils/index-optimization.ts +++ b/packages/db/src/utils/index-optimization.ts @@ -446,9 +446,7 @@ function optimizeSimpleComparison< // - lt/lte: the open lower bound always includes nullish-keyed rows, // so the result is conservatively inexact. const isExact = - operation === `lt` || operation === `lte` - ? false - : queryValue != null + operation === `lt` || operation === `lte` ? false : queryValue != null return { canOptimize: true, matchingKeys, isExact } } From b9cca21a2fd9457983501619574c0e475c019217 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Wed, 24 Jun 2026 11:54:40 +0200 Subject: [PATCH 15/21] fix: avoid locale string range index lookups and re-filter NaN results Two more index-optimization correctness issues: - A BTree index orders strings with localeCompare under the default 'locale' collation, but the WHERE evaluator compares strings with JS relational operators (code-point order). For range predicates these orders disagree (e.g. 'oe-umlaut' > 'z' is true in JS but sorts before 'z' under locale), so an index range lookup can omit matching rows - which re-filtering cannot recover. Locale-backed string range predicates are now left for a full scan (eq/IN use exact equality and are unaffected). - eq/IN against NaN returned isExact: true, but NaN is never equal to itself while the index still returns NaN-keyed rows (SameValueZero map equality). NaN is now treated like a nullish value for exactness, so such results are re-filtered against the full expression. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../index-optimization-partial-and-or.md | 2 + packages/db/src/utils/index-optimization.ts | 120 ++++++++++++++---- 2 files changed, 99 insertions(+), 23 deletions(-) diff --git a/.changeset/index-optimization-partial-and-or.md b/.changeset/index-optimization-partial-and-or.md index a67048c9d7..744f3110da 100644 --- a/.changeset/index-optimization-partial-and-or.md +++ b/.changeset/index-optimization-partial-and-or.md @@ -12,3 +12,5 @@ Fix incorrect results from index-optimized `where` clauses that combine indexed - Strict range comparisons (`gt`/`lt`) on BTree-indexed fields holding normalized values such as dates now correctly exclude the boundary value. - Compound range conditions with a `null`/`undefined` bound (e.g. `gt(score, undefined)`) now re-filter against the full expression instead of returning index-ordered rows, matching the semantics of a full scan (a comparison against `null`/`undefined` is never true). - Index-optimized `eq`, `IN`, and range queries on a field that has rows with `null`/`undefined` values no longer leak those rows into results. BTree indexes store and return such rows (they sort as the smallest key), but a comparison against `null`/`undefined` is never true, so these results are now re-filtered against the full expression to stay equivalent to a full scan. +- Index-optimized `eq` and `IN` against `NaN` no longer leak `NaN`-valued rows. `NaN` is never equal to itself, but indexes still return such rows, so these results are now re-filtered against the full expression. +- String range conditions (`gt`/`gte`/`lt`/`lte`) on a collection using locale string collation (the default) are no longer served by the index. The index orders strings with `localeCompare` while the `where` evaluator compares them with standard relational operators, so an index range lookup could omit matching rows; these conditions now fall back to a full scan. diff --git a/packages/db/src/utils/index-optimization.ts b/packages/db/src/utils/index-optimization.ts index 103cd63557..730f09b043 100644 --- a/packages/db/src/utils/index-optimization.ts +++ b/packages/db/src/utils/index-optimization.ts @@ -102,6 +102,59 @@ export function unionSets(sets: Array>): Set { return result } +/** + * Whether a value can be matched exactly by an index lookup, i.e. the index + * result for it is not a superset that the caller must re-filter. + * + * The WHERE evaluator uses three-valued logic: a comparison against + * `null`/`undefined` yields UNKNOWN, and `NaN` is never equal to itself. BTree + * indexes, however, store and return rows with such values (nullish keys sort + * as the smallest key; `NaN` keys are found via SameValueZero map equality), so + * any result that could include them must be treated as inexact. + */ +function isExactComparisonValue(value: unknown): boolean { + if (value == null) return false + if (typeof value === `number` && Number.isNaN(value)) return false + return true +} + +/** + * Whether the collection orders strings using locale collation. + * + * Under `stringSort: 'locale'` a BTree string index orders values with + * `localeCompare`, but the WHERE evaluator compares strings with JS relational + * operators (code-point order). For range predicates these orders disagree + * (e.g. `'ö' > 'z'` is true in JS but `'ö'` sorts before `'z'` under locale + * `en`), so an index range lookup can omit matching rows. Such omissions cannot + * be recovered by re-filtering, so locale-backed string range predicates must + * not be index-optimized. + */ +function usesLocaleStringSort(collection: CollectionLike): boolean { + const opts = { ...DEFAULT_COMPARE_OPTIONS, ...collection.compareOptions } + return opts.stringSort === `locale` +} + +/** + * Whether a range predicate cannot be safely served by an index because the + * index orders strings with locale collation while the WHERE evaluator uses JS + * relational operators. See {@link usesLocaleStringSort}. + */ +function isLocaleStringRangeUnsafe( + operation: string, + value: unknown, + collection: CollectionLike, +): boolean { + if ( + operation !== `gt` && + operation !== `gte` && + operation !== `lt` && + operation !== `lte` + ) { + return false + } + return typeof value === `string` && usesLocaleStringSort(collection) +} + /** * Optimizes a query expression using available indexes to find matching keys */ @@ -272,6 +325,17 @@ function optimizeCompoundRangeQuery< const fieldPath = fieldKey.split(`.`) const index = findIndexForField(collection, fieldPath) + // A locale-backed string range cannot be served by the index (it may + // omit matching rows that re-filtering cannot recover), so leave this + // field for a full scan instead of collapsing it into a range query. + if ( + operations.some((op) => + isLocaleStringRangeUnsafe(op.operation, op.value, collection), + ) + ) { + continue + } + if (index && index.supports(`gt`) && index.supports(`lt`)) { // Compare values with the same semantics the index uses (dates, // locale strings, ...), in ascending order since bounds are about @@ -293,15 +357,15 @@ function optimizeCompoundRangeQuery< let hasToBound = false let fromInclusive = true let toInclusive = true - // A comparison against null/undefined is never true, but in an index - // those values sort as the smallest key, so a range query cannot - // represent such a bound. Track it and force a re-filter instead of - // claiming the result is exact. - let hasNullBound = false + // A comparison against null/undefined/NaN is never true, but in an + // index nullish values sort as the smallest key (and NaN is retained), + // so a range query cannot represent such a bound. Track it and force a + // re-filter instead of claiming the result is exact. + let hasNonComparableBound = false for (const { operation, value } of operations) { - if (value == null) { - hasNullBound = true + if (!isExactComparisonValue(value)) { + hasNonComparableBound = true continue } switch (operation) { @@ -353,8 +417,9 @@ function optimizeCompoundRangeQuery< // index returns, as they sort as the smallest key). That requires a // non-nullish lower bound to exclude them: without `hasFromBound` // the range is open at the bottom and captures those rows, and a - // nullish bound value (`hasNullBound`) can never bound them out. - isExact: hasFromBound && !hasNullBound, + // non-comparable bound value (`hasNonComparableBound`) can never + // bound them out. + isExact: hasFromBound && !hasNonComparableBound, coveredArgIndices: new Set(operations.map((op) => op.argIndex)), } } @@ -433,20 +498,29 @@ function optimizeSimpleComparison< return { canOptimize: false, matchingKeys: new Set(), isExact: false } } + // A locale-backed string range cannot be served by the index: it may + // omit matching rows, which re-filtering cannot recover. Fall back to a + // full scan instead. + if (isLocaleStringRangeUnsafe(operation, queryValue, collection)) { + return { canOptimize: false, matchingKeys: new Set(), isExact: false } + } + const matchingKeys = index.lookup(indexOperation, queryValue) - // A comparison against null/undefined is never true, but BTree indexes - // store and return rows with a nullish indexed value (they sort as the - // smallest key). Determine whether the index result is exact or a - // superset that the caller must re-filter: - // - eq/gt/gte: a nullish query value matches nothing, while the index - // would still return nullish-keyed rows -> inexact when nullish. A - // non-nullish lower bound (gt/gte) excludes the bottom-sorted nullish - // rows, so those stay exact. + // A comparison against a nullish or NaN value is never true, but BTree + // indexes store and return rows with such values (nullish keys sort as + // the smallest key; NaN keys match via SameValueZero map equality). + // Determine whether the index result is exact or a superset that the + // caller must re-filter: + // - eq/gt/gte: such a query value matches nothing while the index still + // returns those rows -> inexact. A non-nullish lower bound (gt/gte) + // excludes the bottom-sorted nullish rows, so those stay exact. // - lt/lte: the open lower bound always includes nullish-keyed rows, // so the result is conservatively inexact. const isExact = - operation === `lt` || operation === `lte` ? false : queryValue != null + operation === `lt` || operation === `lte` + ? false + : isExactComparisonValue(queryValue) return { canOptimize: true, matchingKeys, isExact } } @@ -638,11 +712,11 @@ function optimizeInArrayExpression< const values = (arrayArg as any).value const index = findIndexForField(collection, fieldPath) - // A nullish member can never be matched by `IN` (a comparison against - // null/undefined is never true), but the index would still return rows - // with a nullish indexed value. When the list contains a nullish member - // the result is a superset that the caller must re-filter. - const isExact = !values.some((value: any) => value == null) + // A nullish or NaN member can never be matched by `IN` (a comparison + // against null/undefined/NaN is never true), but the index would still + // return rows with such an indexed value. When the list contains one of + // those the result is a superset that the caller must re-filter. + const isExact = values.every((value: any) => isExactComparisonValue(value)) if (index) { // Check if the index supports IN operation From d09ec6bc892955af2da0a2e846c629596f9659c2 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 25 Jun 2026 10:27:27 +0200 Subject: [PATCH 16/21] fix: only use indexes for range predicates when ordering is trustworthy Range optimization assumed the index orders values the same way the WHERE evaluator's relational operators do. That holds for numbers, booleans, bigints, lexical strings and valid Dates, but not for: - non-primitive operands (arrays, plain objects, Temporal, invalid Dates), which the evaluator compares via string coercion / identity while the index compares recursively; - indexes created with a custom comparator, whose order is opaque; - fields containing a NaN or invalid Date, which compare equal to every value and break the strict-weak-ordering range traversal relies on. In all three cases an index range lookup can omit genuine matches, which re-filtering cannot recover, so they now fall back to a full scan. The index exposes a supportsRangeOptimization capability (false for custom comparators or when an unorderable value is stored), and the optimizer additionally checks the operand domain. eq/IN are unaffected (exact equality, not ordering). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../index-optimization-partial-and-or.md | 3 + packages/db/src/indexes/base-index.ts | 28 +++++++ packages/db/src/indexes/basic-index.ts | 7 ++ packages/db/src/indexes/btree-index.ts | 7 ++ packages/db/src/indexes/reverse-index.ts | 4 + packages/db/src/utils/index-optimization.ts | 84 ++++++++++++++----- 6 files changed, 111 insertions(+), 22 deletions(-) diff --git a/.changeset/index-optimization-partial-and-or.md b/.changeset/index-optimization-partial-and-or.md index 744f3110da..501ac3b9b1 100644 --- a/.changeset/index-optimization-partial-and-or.md +++ b/.changeset/index-optimization-partial-and-or.md @@ -14,3 +14,6 @@ Fix incorrect results from index-optimized `where` clauses that combine indexed - Index-optimized `eq`, `IN`, and range queries on a field that has rows with `null`/`undefined` values no longer leak those rows into results. BTree indexes store and return such rows (they sort as the smallest key), but a comparison against `null`/`undefined` is never true, so these results are now re-filtered against the full expression to stay equivalent to a full scan. - Index-optimized `eq` and `IN` against `NaN` no longer leak `NaN`-valued rows. `NaN` is never equal to itself, but indexes still return such rows, so these results are now re-filtered against the full expression. - String range conditions (`gt`/`gte`/`lt`/`lte`) on a collection using locale string collation (the default) are no longer served by the index. The index orders strings with `localeCompare` while the `where` evaluator compares them with standard relational operators, so an index range lookup could omit matching rows; these conditions now fall back to a full scan. +- Range conditions whose operand is not ordered the same way by the index and the `where` evaluator (arrays, plain objects, Temporal values, invalid Dates) now fall back to a full scan instead of using the index, which could otherwise omit matching rows. +- Range conditions on an index created with a custom comparator now fall back to a full scan, since the comparator's ordering may not match the `where` evaluator's relational operators. +- Range conditions on a field that contains a `NaN` value (or an invalid Date) now fall back to a full scan. Such values have no well-defined order and break the index traversal, which could otherwise drop genuinely matching rows. diff --git a/packages/db/src/indexes/base-index.ts b/packages/db/src/indexes/base-index.ts index 4346450f90..1c3a8f54da 100644 --- a/packages/db/src/indexes/base-index.ts +++ b/packages/db/src/indexes/base-index.ts @@ -68,6 +68,17 @@ export interface IndexInterface< supports: (operation: IndexOperation) => boolean + /** + * Whether range lookups (gt/gte/lt/lte) on this index can be trusted to + * return every matching key. Range traversal relies on the index ordering, so + * it is unsafe when the index uses a custom comparator (whose order may differ + * from the WHERE evaluator's relational operators) or currently contains a + * value that has no well-defined order (NaN or an invalid Date, which compare + * equal to every value and break the strict-weak-ordering traversal assumes). + * Callers must fall back to a full scan when this is `false`. + */ + get supportsRangeOptimization(): boolean + matchesField: (fieldPath: Array) => boolean matchesCompareOptions: (compareOptions: CompareOptions) => boolean matchesDirection: (direction: OrderByDirection) => boolean @@ -90,6 +101,11 @@ export abstract class BaseIndex< protected totalLookupTime = 0 protected lastUpdated = new Date() protected compareOptions: CompareOptions + /** + * Set by subclasses when constructed with a user-supplied comparator, whose + * ordering may not match the WHERE evaluator's relational operators. + */ + protected hasCustomComparator = false constructor( id: number, @@ -144,6 +160,18 @@ export abstract class BaseIndex< return this.supportedOperations.has(operation) } + get supportsRangeOptimization(): boolean { + return !this.hasCustomComparator && !this.containsUnorderedIndexedValue() + } + + /** + * Whether the index currently holds a value with no well-defined order + * (NaN, or an invalid Date — both normalize to NaN). Such a value compares + * equal to every other value, breaking the strict-weak-ordering that range + * traversal relies on. + */ + protected abstract containsUnorderedIndexedValue(): boolean + matchesField(fieldPath: Array): boolean { return ( this.expression.type === `ref` && diff --git a/packages/db/src/indexes/basic-index.ts b/packages/db/src/indexes/basic-index.ts index b80f7fb439..39efce00a1 100644 --- a/packages/db/src/indexes/basic-index.ts +++ b/packages/db/src/indexes/basic-index.ts @@ -65,6 +65,7 @@ export class BasicIndex< ) { super(id, expression, name, options) this.compareFn = options?.compareFn ?? defaultComparator + this.hasCustomComparator = options?.compareFn != null if (options?.compareOptions) { this.compareOptions = options!.compareOptions } @@ -72,6 +73,12 @@ export class BasicIndex< protected initialize(_options?: BasicIndexOptions): void {} + protected containsUnorderedIndexedValue(): boolean { + // NaN and invalid Dates both normalize to NaN, which collapses to a single + // map bucket (SameValueZero), so a single O(1) check suffices. + return this.valueMap.has(NaN) + } + /** * Adds a value to the index */ diff --git a/packages/db/src/indexes/btree-index.ts b/packages/db/src/indexes/btree-index.ts index b46c707102..fcb7719d7a 100644 --- a/packages/db/src/indexes/btree-index.ts +++ b/packages/db/src/indexes/btree-index.ts @@ -62,6 +62,7 @@ export class BTreeIndex< // Get the base compare function const baseCompareFn = options?.compareFn ?? defaultComparator + this.hasCustomComparator = options?.compareFn != null // Wrap it to denormalize sentinels before comparison // This ensures UNDEFINED_SENTINEL is converted back to undefined @@ -77,6 +78,12 @@ export class BTreeIndex< protected initialize(_options?: BTreeIndexOptions): void {} + protected containsUnorderedIndexedValue(): boolean { + // NaN and invalid Dates both normalize to NaN, which collapses to a single + // map bucket (SameValueZero), so a single O(1) check suffices. + return this.valueMap.has(NaN) + } + /** * Adds a value to the index */ diff --git a/packages/db/src/indexes/reverse-index.ts b/packages/db/src/indexes/reverse-index.ts index 8999b2801c..6ca61636e1 100644 --- a/packages/db/src/indexes/reverse-index.ts +++ b/packages/db/src/indexes/reverse-index.ts @@ -73,6 +73,10 @@ export class ReverseIndex< return this.originalIndex.supports(operation) } + get supportsRangeOptimization(): boolean { + return this.originalIndex.supportsRangeOptimization + } + matchesField(fieldPath: Array): boolean { return this.originalIndex.matchesField(fieldPath) } diff --git a/packages/db/src/utils/index-optimization.ts b/packages/db/src/utils/index-optimization.ts index 730f09b043..a180200ef3 100644 --- a/packages/db/src/utils/index-optimization.ts +++ b/packages/db/src/utils/index-optimization.ts @@ -135,24 +135,55 @@ function usesLocaleStringSort(collection: CollectionLike): boolean { } /** - * Whether a range predicate cannot be safely served by an index because the - * index orders strings with locale collation while the WHERE evaluator uses JS - * relational operators. See {@link usesLocaleStringSort}. + * Whether a range predicate on this operand would use an index ordering that + * differs from the WHERE evaluator's relational operators, so an index range + * lookup could omit genuine matches that re-filtering cannot recover. + * + * The evaluator compares with JS relational operators. That order matches the + * index comparator only for numbers, booleans, bigints, lexically-sorted + * strings and valid Dates. It diverges for locale-sorted strings (localeCompare + * vs code-point order) and for arrays, plain objects, Temporal values, typed + * arrays and invalid Dates (recursive/identity ordering vs string coercion). + * + * Note: `null`/`undefined`/`NaN` operands are not handled here — those are + * superset cases handled by re-filtering ({@link isExactComparisonValue}). */ -function isLocaleStringRangeUnsafe( - operation: string, +function isRangeOrderingDivergent( value: unknown, collection: CollectionLike, ): boolean { - if ( - operation !== `gt` && - operation !== `gte` && - operation !== `lt` && - operation !== `lte` - ) { - return false + switch (typeof value) { + case `number`: + case `bigint`: + case `boolean`: + return false + case `string`: + return usesLocaleStringSort(collection) + case `object`: { + if (value === null) return false + // Only valid Date instances order consistently with the evaluator + return !(value instanceof Date) || Number.isNaN(value.getTime()) + } + default: + return false } - return typeof value === `string` && usesLocaleStringSort(collection) +} + +/** + * Whether a range predicate (gt/gte/lt/lte) on this operand can be safely + * served by the given index: the operand's domain must order the same way the + * index does, and the index itself must support trustworthy range traversal + * (no custom comparator, no stored unorderable value). + */ +function canRangeOptimize( + value: unknown, + index: IndexInterface, + collection: CollectionLike, +): boolean { + return ( + !isRangeOrderingDivergent(value, collection) && + index.supportsRangeOptimization + ) } /** @@ -325,12 +356,14 @@ function optimizeCompoundRangeQuery< const fieldPath = fieldKey.split(`.`) const index = findIndexForField(collection, fieldPath) - // A locale-backed string range cannot be served by the index (it may - // omit matching rows that re-filtering cannot recover), so leave this - // field for a full scan instead of collapsing it into a range query. + // Only collapse this field into a range query when every bound's domain + // orders the same way the index does and the index supports trustworthy + // range traversal. Otherwise the index may omit matching rows that + // re-filtering cannot recover, so leave the field for a full scan. if ( - operations.some((op) => - isLocaleStringRangeUnsafe(op.operation, op.value, collection), + index && + operations.some( + (op) => !canRangeOptimize(op.value, index, collection), ) ) { continue @@ -498,10 +531,17 @@ function optimizeSimpleComparison< return { canOptimize: false, matchingKeys: new Set(), isExact: false } } - // A locale-backed string range cannot be served by the index: it may - // omit matching rows, which re-filtering cannot recover. Fall back to a - // full scan instead. - if (isLocaleStringRangeUnsafe(operation, queryValue, collection)) { + // A range op can only use the index when the operand's domain orders the + // same way the index does and the index supports trustworthy traversal. + // Otherwise the index may omit matching rows, which re-filtering cannot + // recover, so fall back to a full scan. + if ( + (operation === `gt` || + operation === `gte` || + operation === `lt` || + operation === `lte`) && + !canRangeOptimize(queryValue, index, collection) + ) { return { canOptimize: false, matchingKeys: new Set(), isExact: false } } From 8a731a99730588e0312ad450d7e41155cdb0953c Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Thu, 25 Jun 2026 08:29:19 +0000 Subject: [PATCH 17/21] ci: apply automated fixes --- packages/db/src/utils/index-optimization.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/db/src/utils/index-optimization.ts b/packages/db/src/utils/index-optimization.ts index a180200ef3..ad65812ee2 100644 --- a/packages/db/src/utils/index-optimization.ts +++ b/packages/db/src/utils/index-optimization.ts @@ -362,9 +362,7 @@ function optimizeCompoundRangeQuery< // re-filtering cannot recover, so leave the field for a full scan. if ( index && - operations.some( - (op) => !canRangeOptimize(op.value, index, collection), - ) + operations.some((op) => !canRangeOptimize(op.value, index, collection)) ) { continue } From 4b97808d0291884d01c9aa2a9a0e3a72d38f9145 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 25 Jun 2026 11:30:38 +0200 Subject: [PATCH 18/21] fix: give NaN and invalid Dates a stable sort position The comparator returned 0 for NaN against any value (and NaN from invalid-Date subtraction), so NaN had no consistent order. That made ordering by a field containing NaN non-deterministic and, worse, corrupted the strict-weak-ordering that B-tree range traversal relies on, so a stored NaN could make a range query drop genuinely matching rows. ascComparator now places NaN and invalid Dates alongside nulls, giving a well-defined total order. With a valid order the index traversal is correct again, so range queries on a field containing such values no longer deopt to a full scan: NaN simply sorts to the nulls end, where the existing exactness logic excludes it from lower-bounded ranges and re-filters it out of open-bottom (lt/lte) ranges. The index capability therefore only needs to deopt for custom comparators, so the stored-value check is removed. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../index-optimization-partial-and-or.md | 2 +- packages/db/src/indexes/base-index.ts | 18 ++++---------- packages/db/src/indexes/basic-index.ts | 6 ----- packages/db/src/indexes/btree-index.ts | 6 ----- packages/db/src/utils/comparison.ts | 24 +++++++++++++++++++ packages/db/src/utils/index-optimization.ts | 2 +- 6 files changed, 30 insertions(+), 28 deletions(-) diff --git a/.changeset/index-optimization-partial-and-or.md b/.changeset/index-optimization-partial-and-or.md index 501ac3b9b1..536e18aaf3 100644 --- a/.changeset/index-optimization-partial-and-or.md +++ b/.changeset/index-optimization-partial-and-or.md @@ -16,4 +16,4 @@ Fix incorrect results from index-optimized `where` clauses that combine indexed - String range conditions (`gt`/`gte`/`lt`/`lte`) on a collection using locale string collation (the default) are no longer served by the index. The index orders strings with `localeCompare` while the `where` evaluator compares them with standard relational operators, so an index range lookup could omit matching rows; these conditions now fall back to a full scan. - Range conditions whose operand is not ordered the same way by the index and the `where` evaluator (arrays, plain objects, Temporal values, invalid Dates) now fall back to a full scan instead of using the index, which could otherwise omit matching rows. - Range conditions on an index created with a custom comparator now fall back to a full scan, since the comparator's ordering may not match the `where` evaluator's relational operators. -- Range conditions on a field that contains a `NaN` value (or an invalid Date) now fall back to a full scan. Such values have no well-defined order and break the index traversal, which could otherwise drop genuinely matching rows. +- `NaN` and invalid Dates now have a well-defined sort position (alongside nulls) instead of comparing equal to every value. This fixes non-deterministic ordering when sorting by a field that contains such values, and — because the index ordering is no longer corrupted by them — range queries on a field that contains a `NaN` or invalid Date are once again served by the index (the non-matching value is removed by the existing re-filter) rather than dropping genuinely matching rows. diff --git a/packages/db/src/indexes/base-index.ts b/packages/db/src/indexes/base-index.ts index 1c3a8f54da..945221e6fa 100644 --- a/packages/db/src/indexes/base-index.ts +++ b/packages/db/src/indexes/base-index.ts @@ -71,11 +71,9 @@ export interface IndexInterface< /** * Whether range lookups (gt/gte/lt/lte) on this index can be trusted to * return every matching key. Range traversal relies on the index ordering, so - * it is unsafe when the index uses a custom comparator (whose order may differ - * from the WHERE evaluator's relational operators) or currently contains a - * value that has no well-defined order (NaN or an invalid Date, which compare - * equal to every value and break the strict-weak-ordering traversal assumes). - * Callers must fall back to a full scan when this is `false`. + * it is unsafe when the index uses a custom comparator, whose order may not + * match the WHERE evaluator's relational operators. Callers must fall back to + * a full scan when this is `false`. */ get supportsRangeOptimization(): boolean @@ -161,17 +159,9 @@ export abstract class BaseIndex< } get supportsRangeOptimization(): boolean { - return !this.hasCustomComparator && !this.containsUnorderedIndexedValue() + return !this.hasCustomComparator } - /** - * Whether the index currently holds a value with no well-defined order - * (NaN, or an invalid Date — both normalize to NaN). Such a value compares - * equal to every other value, breaking the strict-weak-ordering that range - * traversal relies on. - */ - protected abstract containsUnorderedIndexedValue(): boolean - matchesField(fieldPath: Array): boolean { return ( this.expression.type === `ref` && diff --git a/packages/db/src/indexes/basic-index.ts b/packages/db/src/indexes/basic-index.ts index 39efce00a1..b9b06d1925 100644 --- a/packages/db/src/indexes/basic-index.ts +++ b/packages/db/src/indexes/basic-index.ts @@ -73,12 +73,6 @@ export class BasicIndex< protected initialize(_options?: BasicIndexOptions): void {} - protected containsUnorderedIndexedValue(): boolean { - // NaN and invalid Dates both normalize to NaN, which collapses to a single - // map bucket (SameValueZero), so a single O(1) check suffices. - return this.valueMap.has(NaN) - } - /** * Adds a value to the index */ diff --git a/packages/db/src/indexes/btree-index.ts b/packages/db/src/indexes/btree-index.ts index fcb7719d7a..8b92095f01 100644 --- a/packages/db/src/indexes/btree-index.ts +++ b/packages/db/src/indexes/btree-index.ts @@ -78,12 +78,6 @@ export class BTreeIndex< protected initialize(_options?: BTreeIndexOptions): void {} - protected containsUnorderedIndexedValue(): boolean { - // NaN and invalid Dates both normalize to NaN, which collapses to a single - // map bucket (SameValueZero), so a single O(1) check suffices. - return this.valueMap.has(NaN) - } - /** * Adds a value to the index */ diff --git a/packages/db/src/utils/comparison.ts b/packages/db/src/utils/comparison.ts index bf5ac1a913..5c461c3f22 100644 --- a/packages/db/src/utils/comparison.ts +++ b/packages/db/src/utils/comparison.ts @@ -17,6 +17,19 @@ function getObjectId(obj: object): number { return id } +/** + * Whether a value has no natural order: `NaN`, or an invalid Date (whose + * timestamp is `NaN`). Such values cannot participate in a total order, so the + * comparator handles them explicitly instead of letting them compare equal to + * everything (`NaN`) or produce a `NaN` result (invalid Date subtraction). + */ +function isUnorderable(value: any): boolean { + return ( + (typeof value === `number` && Number.isNaN(value)) || + (value instanceof Date && Number.isNaN(value.getTime())) + ) +} + /** * Universal comparison function for all data types * Handles null/undefined, strings, arrays, dates, objects, and primitives @@ -30,6 +43,17 @@ export const ascComparator = (a: any, b: any, opts: CompareOptions): number => { if (a == null) return nulls === `first` ? -1 : 1 if (b == null) return nulls === `first` ? 1 : -1 + // Handle values with no natural order (NaN, invalid Dates). They would + // otherwise compare equal to everything (NaN) or yield NaN from the date + // subtraction below, breaking the total order the rest of the system relies + // on (e.g. B-tree range traversal). Give them a stable position alongside + // nulls so the ordering stays well-defined. + const aUnordered = isUnorderable(a) + const bUnordered = isUnorderable(b) + if (aUnordered && bUnordered) return 0 + if (aUnordered) return nulls === `first` ? -1 : 1 + if (bUnordered) return nulls === `first` ? 1 : -1 + // if a and b are both strings, compare them based on locale if (typeof a === `string` && typeof b === `string`) { if (opts.stringSort === `locale`) { diff --git a/packages/db/src/utils/index-optimization.ts b/packages/db/src/utils/index-optimization.ts index ad65812ee2..07360341f7 100644 --- a/packages/db/src/utils/index-optimization.ts +++ b/packages/db/src/utils/index-optimization.ts @@ -173,7 +173,7 @@ function isRangeOrderingDivergent( * Whether a range predicate (gt/gte/lt/lte) on this operand can be safely * served by the given index: the operand's domain must order the same way the * index does, and the index itself must support trustworthy range traversal - * (no custom comparator, no stored unorderable value). + * (no custom comparator). */ function canRangeOptimize( value: unknown, From 35fb96da06065aed6f9708a7718955f69e88f041 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Fri, 26 Jun 2026 11:40:21 +0200 Subject: [PATCH 19/21] feat: adopt PostgreSQL float semantics for NaN (supersedes #1617) Fold the NaN/invalid-Date semantics from the #1617 proposal into this PR so the index-optimization correctness work is built on the final, coherent contract instead of the interim JS-semantics handling. Previously this branch gave NaN/invalid Dates a stable sort position *for sorting only* while the WHERE evaluator still rejected them (NaN != NaN), relying on re-filtering to drop index-returned NaN rows. That left a JS/SQL hybrid where joins/groupBy/distinct (which match NaN = NaN via the hash index) disagreed with WHERE/ordering. Following PostgreSQL, NaN (and invalid Dates, whose timestamp is NaN) is now equal to itself and greater than every other non-null value: - comparison.ts: ascComparator orders NaN/invalid Dates as the greatest non-null value (was: alongside nulls); isUnorderable is exported. - evaluators.ts: eq/gt/gte/lt/lte/in implement the same via valuesEqual. - index-optimization.ts: because the index and evaluator now agree on NaN/invalid Dates, they are treated as exact (no re-filter) and invalid Dates are no longer range-divergent. This resolves the reviewer's invalid-Date eq/IN issue (indexed == full-scan) and simplifies the NaN-specific defensiveness down to the remaining nullish cases. null/undefined are unchanged: still three-valued logic (UNKNOWN). Tests: new nan-semantics.test.ts (numeric NaN + invalid Date, asserting indexed == full-scan), PG-semantics comparison.test.ts and evaluator NaN block; updated the NaN tests in collection-indexes.test.ts and deterministic-ordering.test.ts that encoded the old JS behavior. Docs and a minor changeset added. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../index-optimization-partial-and-or.md | 4 +- .changeset/nan-postgres-semantics.md | 15 ++ docs/guides/live-queries.md | 7 + packages/db/src/query/compiler/evaluators.ts | 40 +++- packages/db/src/utils/comparison.ts | 25 +-- packages/db/src/utils/index-optimization.ts | 60 +++--- packages/db/tests/collection-indexes.test.ts | 36 ++-- packages/db/tests/comparison.test.ts | 37 ++-- .../db/tests/deterministic-ordering.test.ts | 6 +- packages/db/tests/nan-semantics.test.ts | 195 ++++++++++++++++++ .../tests/query/compiler/evaluators.test.ts | 81 ++++++++ 11 files changed, 427 insertions(+), 79 deletions(-) create mode 100644 .changeset/nan-postgres-semantics.md create mode 100644 packages/db/tests/nan-semantics.test.ts diff --git a/.changeset/index-optimization-partial-and-or.md b/.changeset/index-optimization-partial-and-or.md index 536e18aaf3..b5b6067360 100644 --- a/.changeset/index-optimization-partial-and-or.md +++ b/.changeset/index-optimization-partial-and-or.md @@ -12,8 +12,6 @@ Fix incorrect results from index-optimized `where` clauses that combine indexed - Strict range comparisons (`gt`/`lt`) on BTree-indexed fields holding normalized values such as dates now correctly exclude the boundary value. - Compound range conditions with a `null`/`undefined` bound (e.g. `gt(score, undefined)`) now re-filter against the full expression instead of returning index-ordered rows, matching the semantics of a full scan (a comparison against `null`/`undefined` is never true). - Index-optimized `eq`, `IN`, and range queries on a field that has rows with `null`/`undefined` values no longer leak those rows into results. BTree indexes store and return such rows (they sort as the smallest key), but a comparison against `null`/`undefined` is never true, so these results are now re-filtered against the full expression to stay equivalent to a full scan. -- Index-optimized `eq` and `IN` against `NaN` no longer leak `NaN`-valued rows. `NaN` is never equal to itself, but indexes still return such rows, so these results are now re-filtered against the full expression. - String range conditions (`gt`/`gte`/`lt`/`lte`) on a collection using locale string collation (the default) are no longer served by the index. The index orders strings with `localeCompare` while the `where` evaluator compares them with standard relational operators, so an index range lookup could omit matching rows; these conditions now fall back to a full scan. -- Range conditions whose operand is not ordered the same way by the index and the `where` evaluator (arrays, plain objects, Temporal values, invalid Dates) now fall back to a full scan instead of using the index, which could otherwise omit matching rows. +- Range conditions whose operand is not ordered the same way by the index and the `where` evaluator (arrays, plain objects, Temporal values) now fall back to a full scan instead of using the index, which could otherwise omit matching rows. - Range conditions on an index created with a custom comparator now fall back to a full scan, since the comparator's ordering may not match the `where` evaluator's relational operators. -- `NaN` and invalid Dates now have a well-defined sort position (alongside nulls) instead of comparing equal to every value. This fixes non-deterministic ordering when sorting by a field that contains such values, and — because the index ordering is no longer corrupted by them — range queries on a field that contains a `NaN` or invalid Date are once again served by the index (the non-matching value is removed by the existing re-filter) rather than dropping genuinely matching rows. diff --git a/.changeset/nan-postgres-semantics.md b/.changeset/nan-postgres-semantics.md new file mode 100644 index 0000000000..63d7d1b2b9 --- /dev/null +++ b/.changeset/nan-postgres-semantics.md @@ -0,0 +1,15 @@ +--- +'@tanstack/db': minor +--- + +Adopt PostgreSQL float semantics for `NaN` in `where` clauses and ordering. + +`NaN` (and invalid `Date` values, whose timestamp is `NaN`) previously had no consistent order — `NaN === NaN` is `false` in JavaScript, so `NaN` compared unequal to everything and could not be sorted or indexed deterministically. Following PostgreSQL, `NaN` is now treated as **equal to itself** and **greater than every other non-null value**: + +- `eq(row.value, NaN)` matches rows whose value is `NaN`; `inArray(row.value, [NaN, ...])` matches them too. +- Range comparisons treat `NaN` as the greatest value: `gt`/`gte` include it, `lt`/`lte` exclude it. +- Ordering by a field containing `NaN` is now deterministic, with `NaN` sorting last (and `null` still ordered by `NULLS FIRST`/`NULLS LAST`). + +`null`/`undefined` are unaffected: they continue to use three-valued logic (a comparison with `null` yields `UNKNOWN`). + +This makes results independent of whether a query is served from an index or a full scan. diff --git a/docs/guides/live-queries.md b/docs/guides/live-queries.md index 0780871a3b..4e8ec8dcbf 100644 --- a/docs/guides/live-queries.md +++ b/docs/guides/live-queries.md @@ -728,6 +728,13 @@ not(condition) For a complete reference of all available functions, see the [Expression Functions Reference](#expression-functions-reference) section. +### Comparison semantics + +Comparisons follow SQL/PostgreSQL conventions rather than raw JavaScript: + +- **`null` / `undefined` use three-valued logic.** Any comparison involving `null` or `undefined` evaluates to `UNKNOWN`, so the row is not matched. For example `eq(user.score, null)` matches nothing — use a dedicated null check (e.g. `isUndefined`) to match missing values. +- **`NaN` follows PostgreSQL float semantics.** `NaN` is treated as equal to itself and greater than every other (non-null) value. So `eq(row.value, NaN)` matches `NaN` rows, `gt(row.value, x)` includes `NaN`, and ordering by such a field places `NaN` last. (Invalid `Date` values, whose timestamp is `NaN`, behave the same way.) This differs from JavaScript, where `NaN === NaN` is `false`, and matches how PostgreSQL orders and indexes floating-point values. + ## Select Use `select` to specify which fields to include in your results and transform your data. Without `select`, you get the full schema. diff --git a/packages/db/src/query/compiler/evaluators.ts b/packages/db/src/query/compiler/evaluators.ts index 929ac56dfb..fa2e90725d 100644 --- a/packages/db/src/query/compiler/evaluators.ts +++ b/packages/db/src/query/compiler/evaluators.ts @@ -3,7 +3,11 @@ import { UnknownExpressionTypeError, UnknownFunctionError, } from '../../errors.js' -import { areValuesEqual, normalizeValue } from '../../utils/comparison.js' +import { + areValuesEqual, + isUnorderable, + normalizeValue, +} from '../../utils/comparison.js' import type { BasicExpression, Func, PropRef } from '../ir.js' import type { NamespacedRow } from '../../types.js' @@ -14,6 +18,19 @@ function isUnknown(value: any): boolean { return value === null || value === undefined } +/** + * Equality that follows PostgreSQL float semantics for `NaN`/invalid Dates: + * such values are equal to one another and unequal to anything else. For all + * other values it defers to {@link areValuesEqual}. Operands must not be + * null/undefined (callers handle UNKNOWN first). + */ +function valuesEqual(a: any, b: any): boolean { + if (isUnorderable(a) || isUnorderable(b)) { + return isUnorderable(a) && isUnorderable(b) + } + return areValuesEqual(a, b) +} + function toDateValue(value: any): Date | null { if (value instanceof Date) { return Number.isNaN(value.getTime()) ? null : value @@ -233,8 +250,9 @@ function compileFunction(func: Func, isSingleRow: boolean): (data: any) => any { if (isUnknown(a) || isUnknown(b)) { return null } - // Use areValuesEqual for proper Uint8Array/Buffer comparison - return areValuesEqual(a, b) + // NaN/invalid Dates are equal to one another (PostgreSQL semantics); + // otherwise use areValuesEqual for proper Uint8Array/Buffer comparison + return valuesEqual(a, b) } } case `gt`: { @@ -247,6 +265,11 @@ function compileFunction(func: Func, isSingleRow: boolean): (data: any) => any { if (isUnknown(a) || isUnknown(b)) { return null } + // NaN/invalid Dates sort greater than every other value, and are equal + // to one another (PostgreSQL semantics) + if (isUnorderable(a) || isUnorderable(b)) { + return isUnorderable(a) && !isUnorderable(b) + } return a > b } } @@ -260,6 +283,9 @@ function compileFunction(func: Func, isSingleRow: boolean): (data: any) => any { if (isUnknown(a) || isUnknown(b)) { return null } + if (isUnorderable(a) || isUnorderable(b)) { + return isUnorderable(a) + } return a >= b } } @@ -273,6 +299,9 @@ function compileFunction(func: Func, isSingleRow: boolean): (data: any) => any { if (isUnknown(a) || isUnknown(b)) { return null } + if (isUnorderable(a) || isUnorderable(b)) { + return isUnorderable(b) && !isUnorderable(a) + } return a < b } } @@ -286,6 +315,9 @@ function compileFunction(func: Func, isSingleRow: boolean): (data: any) => any { if (isUnknown(a) || isUnknown(b)) { return null } + if (isUnorderable(a) || isUnorderable(b)) { + return isUnorderable(b) + } return a <= b } } @@ -370,7 +402,7 @@ function compileFunction(func: Func, isSingleRow: boolean): (data: any) => any { if (!Array.isArray(array)) { return false } - return array.some((item) => normalizeValue(item) === value) + return array.some((item) => valuesEqual(normalizeValue(item), value)) } } diff --git a/packages/db/src/utils/comparison.ts b/packages/db/src/utils/comparison.ts index 5c461c3f22..992f0098c5 100644 --- a/packages/db/src/utils/comparison.ts +++ b/packages/db/src/utils/comparison.ts @@ -18,12 +18,14 @@ function getObjectId(obj: object): number { } /** - * Whether a value has no natural order: `NaN`, or an invalid Date (whose - * timestamp is `NaN`). Such values cannot participate in a total order, so the - * comparator handles them explicitly instead of letting them compare equal to - * everything (`NaN`) or produce a `NaN` result (invalid Date subtraction). + * Whether a value has no IEEE-754 natural order: `NaN`, or an invalid Date + * (whose timestamp is `NaN`). The query engine follows PostgreSQL float + * semantics for these values — they are all equal to one another and greater + * than every other (non-null) value — so the comparator and the WHERE + * evaluator treat them explicitly instead of letting `NaN` compare unequal to + * everything (which has no consistent order and cannot be indexed or sorted). */ -function isUnorderable(value: any): boolean { +export function isUnorderable(value: any): boolean { return ( (typeof value === `number` && Number.isNaN(value)) || (value instanceof Date && Number.isNaN(value.getTime())) @@ -43,16 +45,15 @@ export const ascComparator = (a: any, b: any, opts: CompareOptions): number => { if (a == null) return nulls === `first` ? -1 : 1 if (b == null) return nulls === `first` ? 1 : -1 - // Handle values with no natural order (NaN, invalid Dates). They would - // otherwise compare equal to everything (NaN) or yield NaN from the date - // subtraction below, breaking the total order the rest of the system relies - // on (e.g. B-tree range traversal). Give them a stable position alongside - // nulls so the ordering stays well-defined. + // Handle NaN / invalid Dates. Following PostgreSQL float semantics, they are + // all equal and sort greater than every other non-null value. This keeps the + // order total (NaN would otherwise compare equal to everything), so such + // values can be sorted and stored in tree-based indexes. const aUnordered = isUnorderable(a) const bUnordered = isUnorderable(b) if (aUnordered && bUnordered) return 0 - if (aUnordered) return nulls === `first` ? -1 : 1 - if (bUnordered) return nulls === `first` ? 1 : -1 + if (aUnordered) return 1 + if (bUnordered) return -1 // if a and b are both strings, compare them based on locale if (typeof a === `string` && typeof b === `string`) { diff --git a/packages/db/src/utils/index-optimization.ts b/packages/db/src/utils/index-optimization.ts index 07360341f7..5a52a5ec54 100644 --- a/packages/db/src/utils/index-optimization.ts +++ b/packages/db/src/utils/index-optimization.ts @@ -106,16 +106,17 @@ export function unionSets(sets: Array>): Set { * Whether a value can be matched exactly by an index lookup, i.e. the index * result for it is not a superset that the caller must re-filter. * - * The WHERE evaluator uses three-valued logic: a comparison against - * `null`/`undefined` yields UNKNOWN, and `NaN` is never equal to itself. BTree - * indexes, however, store and return rows with such values (nullish keys sort - * as the smallest key; `NaN` keys are found via SameValueZero map equality), so - * any result that could include them must be treated as inexact. + * Only `null`/`undefined` are inexact: the WHERE evaluator's three-valued logic + * makes any comparison against them UNKNOWN, yet a BTree index stores and + * returns rows with nullish keys (they sort to the nulls end), so a result that + * could include such rows must be re-filtered. + * + * `NaN` and invalid Dates are exact: under the engine's PostgreSQL float + * semantics they are equal to themselves and ordered (greatest non-null value), + * so the evaluator and the index agree on them and no re-filtering is needed. */ function isExactComparisonValue(value: unknown): boolean { - if (value == null) return false - if (typeof value === `number` && Number.isNaN(value)) return false - return true + return value != null } /** @@ -139,14 +140,16 @@ function usesLocaleStringSort(collection: CollectionLike): boolean { * differs from the WHERE evaluator's relational operators, so an index range * lookup could omit genuine matches that re-filtering cannot recover. * - * The evaluator compares with JS relational operators. That order matches the - * index comparator only for numbers, booleans, bigints, lexically-sorted - * strings and valid Dates. It diverges for locale-sorted strings (localeCompare - * vs code-point order) and for arrays, plain objects, Temporal values, typed - * arrays and invalid Dates (recursive/identity ordering vs string coercion). + * The evaluator compares with JS relational operators (extended with + * PostgreSQL float semantics for `NaN`/invalid Dates). That order matches the + * index comparator for numbers, booleans, bigints, lexically-sorted strings, + * Dates (valid, ordered by time; invalid, ordered as the greatest value) and + * `NaN`. It diverges for locale-sorted strings (localeCompare vs code-point + * order) and for arrays, plain objects, Temporal values and typed arrays + * (recursive/identity ordering vs string coercion). * - * Note: `null`/`undefined`/`NaN` operands are not handled here — those are - * superset cases handled by re-filtering ({@link isExactComparisonValue}). + * Note: `null`/`undefined` operands are not handled here — those are superset + * cases handled by re-filtering ({@link isExactComparisonValue}). */ function isRangeOrderingDivergent( value: unknown, @@ -161,8 +164,9 @@ function isRangeOrderingDivergent( return usesLocaleStringSort(collection) case `object`: { if (value === null) return false - // Only valid Date instances order consistently with the evaluator - return !(value instanceof Date) || Number.isNaN(value.getTime()) + // Dates order consistently with the evaluator: valid Dates by time, and + // invalid Dates as the greatest value under PostgreSQL float semantics. + return !(value instanceof Date) } default: return false @@ -388,10 +392,11 @@ function optimizeCompoundRangeQuery< let hasToBound = false let fromInclusive = true let toInclusive = true - // A comparison against null/undefined/NaN is never true, but in an - // index nullish values sort as the smallest key (and NaN is retained), - // so a range query cannot represent such a bound. Track it and force a - // re-filter instead of claiming the result is exact. + // A comparison against null/undefined is never true, but in an index + // nullish values sort to the nulls end, so a range query cannot + // represent such a bound. Track it and force a re-filter instead of + // claiming the result is exact. (NaN/invalid Dates are ordered and + // comparable under PostgreSQL semantics, so they are real bounds.) let hasNonComparableBound = false for (const { operation, value } of operations) { @@ -545,16 +550,17 @@ function optimizeSimpleComparison< const matchingKeys = index.lookup(indexOperation, queryValue) - // A comparison against a nullish or NaN value is never true, but BTree - // indexes store and return rows with such values (nullish keys sort as - // the smallest key; NaN keys match via SameValueZero map equality). + // A comparison against a nullish value is never true, but BTree indexes + // store and return rows with nullish keys (they sort to the nulls end). // Determine whether the index result is exact or a superset that the // caller must re-filter: - // - eq/gt/gte: such a query value matches nothing while the index still - // returns those rows -> inexact. A non-nullish lower bound (gt/gte) - // excludes the bottom-sorted nullish rows, so those stay exact. + // - eq/gt/gte: a nullish query value matches nothing while the index + // still returns nullish-keyed rows -> inexact. A non-nullish lower + // bound (gt/gte) excludes those bottom-sorted rows, so they stay exact. // - lt/lte: the open lower bound always includes nullish-keyed rows, // so the result is conservatively inexact. + // NaN/invalid Dates are exact here: under PostgreSQL float semantics the + // evaluator and the index agree on them (equal to self, greatest). const isExact = operation === `lt` || operation === `lte` ? false diff --git a/packages/db/tests/collection-indexes.test.ts b/packages/db/tests/collection-indexes.test.ts index a9bb013efb..3a136fa96c 100644 --- a/packages/db/tests/collection-indexes.test.ts +++ b/packages/db/tests/collection-indexes.test.ts @@ -1403,9 +1403,10 @@ describe(`Collection Indexes`, () => { expect(names).toEqual([`ö`]) }) - it(`should not match a row with a NaN value for an equality on NaN`, async () => { - // NaN is never equal to itself, so `eq(score, NaN)` must return no rows - // even though the index stores and can return the NaN-valued row. + it(`should match a row with a NaN value for an equality on NaN`, async () => { + // Under PostgreSQL float semantics NaN is equal to itself, so + // `eq(score, NaN)` matches the NaN-valued row (and the index, which + // stores and returns it, agrees with a full scan). const nanCollection = createCollection< { id: string; score: number }, string @@ -1431,13 +1432,15 @@ describe(`Collection Indexes`, () => { where: eq(new PropRef([`score`]), NaN), })! - expect(result).toEqual([]) + const ids = result.map((r) => r.value.id).sort() + expect(ids).toEqual([`2`]) }) - it(`should not match a row with a NaN value for an IN list containing NaN`, async () => { - // A row only matches `IN` when its value equals a listed value, and NaN - // is never equal to itself. So `inArray(score, [NaN, 5])` must match - // only the row with score 5 and never the NaN-valued row. + it(`should match a row with a NaN value for an IN list containing NaN`, async () => { + // A row matches `IN` when its value equals a listed value. Under + // PostgreSQL float semantics NaN is equal to itself, so + // `inArray(score, [NaN, 5])` matches both the score-5 row and the + // NaN-valued row. const nanCollection = createCollection< { id: string; score: number }, string @@ -1464,7 +1467,7 @@ describe(`Collection Indexes`, () => { })! const ids = result.map((r) => r.value.id).sort() - expect(ids).toEqual([`1`]) + expect(ids).toEqual([`1`, `2`]) }) it(`should return array-valued rows for a range predicate consistently with a full scan`, async () => { @@ -1537,8 +1540,9 @@ describe(`Collection Indexes`, () => { it(`should return all matching rows for a range predicate when the field also contains NaN`, async () => { // A range predicate must return every matching row even when other rows - // hold a NaN value for the field. With scores NaN, 1, 3, 5 and 7, - // `score > 2` matches the rows with scores 3, 5 and 7. + // hold a NaN value for the field. Under PostgreSQL float semantics NaN is + // the greatest value, so with scores NaN, 1, 3, 5 and 7, `score > 2` + // matches the rows with scores 3, 5, 7 and NaN. const nanCollection = createCollection< { id: string; score: number }, string @@ -1568,13 +1572,13 @@ describe(`Collection Indexes`, () => { })! const ids = result.map((r) => r.value.id).sort() - expect(ids).toEqual([`five`, `seven`, `three`]) + expect(ids).toEqual([`five`, `nan`, `seven`, `three`]) }) it(`should use the index for a range query on a field that also contains NaN`, async () => { - // A NaN value has a well-defined sort position (with nulls), so a range - // query on the field can still be served by the index and does not need - // to fall back to a full scan. + // A NaN value has a well-defined sort position (greatest, under + // PostgreSQL float semantics), so a range query on the field can still be + // served by the index and does not need to fall back to a full scan. const nanCollection = createCollection< { id: string; score: number }, string @@ -1605,7 +1609,7 @@ describe(`Collection Indexes`, () => { })! const ids = result.map((r) => r.value.id).sort() - expect(ids).toEqual([`five`, `seven`, `three`]) + expect(ids).toEqual([`five`, `nan`, `seven`, `three`]) expectIndexUsage(tracker.stats, { shouldUseIndex: true, diff --git a/packages/db/tests/comparison.test.ts b/packages/db/tests/comparison.test.ts index a70c38981f..ae40488f43 100644 --- a/packages/db/tests/comparison.test.ts +++ b/packages/db/tests/comparison.test.ts @@ -2,31 +2,40 @@ import { describe, expect, it } from 'vitest' import { ascComparator, defaultComparator } from '../src/utils/comparison' import { DEFAULT_COMPARE_OPTIONS } from '../src/utils' -describe(`ascComparator with values that have no natural order`, () => { +describe(`ascComparator - PostgreSQL float semantics for NaN`, () => { const opts = DEFAULT_COMPARE_OPTIONS // nulls: `first` - it(`should order NaN consistently relative to numbers`, () => { - // NaN has no natural order, but the comparator must still place it - // consistently (alongside nulls, which sort first by default) so the - // overall ordering stays well-defined. - expect(ascComparator(NaN, 5, opts)).toBeLessThan(0) - expect(ascComparator(5, NaN, opts)).toBeGreaterThan(0) + it(`orders NaN greater than every number`, () => { + expect(ascComparator(NaN, 5, opts)).toBeGreaterThan(0) + expect(ascComparator(5, NaN, opts)).toBeLessThan(0) + }) + + it(`treats NaN as equal to NaN`, () => { expect(ascComparator(NaN, NaN, opts)).toBe(0) }) - it(`should produce a stable total order when sorting numbers that include NaN`, () => { + it(`produces a stable total order with NaN sorting last`, () => { const sorted = [3, NaN, 1, 5, NaN].sort((a, b) => defaultComparator(a, b)) - // NaN values sort to the front (same end as nulls), the rest ascending - expect(sorted.slice(0, 2).every((v) => Number.isNaN(v))).toBe(true) - expect(sorted.slice(2)).toEqual([1, 3, 5]) + expect(sorted.slice(0, 3)).toEqual([1, 3, 5]) + expect(sorted.slice(3).every((v) => Number.isNaN(v))).toBe(true) + }) + + it(`keeps null before non-null values regardless of NaN`, () => { + // nulls still sort first by default; NaN sorts last (greatest non-null) + const sorted = [5, NaN, null, 1].sort((a, b) => defaultComparator(a, b)) + + expect(sorted[0]).toBe(null) + expect(sorted[1]).toBe(1) + expect(sorted[2]).toBe(5) + expect(Number.isNaN(sorted[3])).toBe(true) }) - it(`should order an invalid Date consistently relative to valid Dates`, () => { + it(`orders an invalid Date greater than valid Dates`, () => { const invalid = new Date(`not a date`) const valid = new Date(`2023-01-01`) - expect(ascComparator(invalid, valid, opts)).toBeLessThan(0) - expect(ascComparator(valid, invalid, opts)).toBeGreaterThan(0) + expect(ascComparator(invalid, valid, opts)).toBeGreaterThan(0) + expect(ascComparator(valid, invalid, opts)).toBeLessThan(0) }) }) diff --git a/packages/db/tests/deterministic-ordering.test.ts b/packages/db/tests/deterministic-ordering.test.ts index 2714688872..160d03284b 100644 --- a/packages/db/tests/deterministic-ordering.test.ts +++ b/packages/db/tests/deterministic-ordering.test.ts @@ -517,10 +517,10 @@ describe(`Deterministic Ordering`, () => { ], }) - // NaN has no natural order; it sorts with nulls (first), then the - // numbers ascending. + // Under PostgreSQL float semantics NaN is the greatest value, so the + // numbers sort ascending first and NaN sorts last. const keys = changes?.map((c) => c.key) - expect(keys).toEqual([`nan`, `b`, `c`, `a`]) + expect(keys).toEqual([`b`, `c`, `a`, `nan`]) }) }) }) diff --git a/packages/db/tests/nan-semantics.test.ts b/packages/db/tests/nan-semantics.test.ts new file mode 100644 index 0000000000..6acca7fe3d --- /dev/null +++ b/packages/db/tests/nan-semantics.test.ts @@ -0,0 +1,195 @@ +import { describe, expect, it } from 'vitest' +import { createCollection } from '../src/collection/index.js' +import { eq, gt, inArray, lt } from '../src/query/builder/functions' +import { PropRef } from '../src/query/ir' +import { BTreeIndex } from '../src/indexes/btree-index.js' +import type { BasicExpression } from '../src/query/ir' + +/** + * The query engine follows PostgreSQL float semantics for `NaN`: it is equal to + * itself and greater than every other (non-null) value, so it has a + * well-defined order and behaves like a normal value in `where` clauses and + * `orderBy`. These tests also assert that querying with an index produces the + * same result as a full scan. + */ +interface Row { + id: string + score: number +} + +const data: Array = [ + { id: `nan`, score: NaN }, + { id: `one`, score: 1 }, + { id: `three`, score: 3 }, + { id: `five`, score: 5 }, + { id: `seven`, score: 7 }, +] + +function makeCollection() { + const collection = createCollection({ + getKey: (row) => row.id, + startSync: true, + autoIndex: `off`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + for (const row of data) write({ type: `insert`, value: row }) + commit() + markReady() + }, + }, + }) + return collection +} + +async function queryBothPaths(where: BasicExpression) { + const indexed = makeCollection() + await indexed.stateWhenReady() + indexed.createIndex((row) => row.score) + const fullScan = makeCollection() + await fullScan.stateWhenReady() + + const indexedIds = indexed + .currentStateAsChanges({ where })! + .map((c) => c.value.id) + .sort() + const fullScanIds = fullScan + .currentStateAsChanges({ where })! + .map((c) => c.value.id) + .sort() + + // The chosen execution strategy must not change the result + expect(indexedIds).toEqual(fullScanIds) + return indexedIds +} + +describe(`NaN query semantics (PostgreSQL float semantics)`, () => { + it(`matches NaN rows for equality on NaN`, async () => { + expect(await queryBothPaths(eq(new PropRef([`score`]), NaN))).toEqual([ + `nan`, + ]) + }) + + it(`treats NaN as greater than every number in a range query`, async () => { + // score > 2 matches 3, 5, 7 and NaN (NaN is greatest) + expect(await queryBothPaths(gt(new PropRef([`score`]), 2))).toEqual([ + `five`, + `nan`, + `seven`, + `three`, + ]) + }) + + it(`excludes NaN from a less-than range query`, async () => { + // score < 4 matches 1 and 3, but not NaN (NaN is greatest) + expect(await queryBothPaths(lt(new PropRef([`score`]), 4))).toEqual([ + `one`, + `three`, + ]) + }) + + it(`matches a NaN member of an IN list`, async () => { + expect( + await queryBothPaths(inArray(new PropRef([`score`]), [NaN, 3])), + ).toEqual([`nan`, `three`]) + }) + + it(`orders NaN last when sorting ascending`, async () => { + const collection = makeCollection() + await collection.stateWhenReady() + + const ordered = collection + .currentStateAsChanges({ + orderBy: [ + { + expression: new PropRef([`score`]), + compareOptions: { direction: `asc`, nulls: `first` }, + }, + ], + })! + .map((c) => c.value.id) + + expect(ordered).toEqual([`one`, `three`, `five`, `seven`, `nan`]) + }) +}) + +/** + * Invalid Dates have a `NaN` timestamp, so they follow the same PostgreSQL + * float semantics as `NaN`: equal to one another and greater than every valid + * Date. The indexed and full-scan paths must agree. + */ +describe(`invalid Date query semantics (PostgreSQL float semantics)`, () => { + interface DateRow { + id: string + createdAt: Date + } + + const valid = new Date(`2023-01-01`) + const dateData: Array = [ + { id: `invalid`, createdAt: new Date(`not a date`) }, + { id: `valid`, createdAt: valid }, + ] + + function makeDateCollection() { + return createCollection({ + getKey: (row) => row.id, + startSync: true, + autoIndex: `off`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + for (const row of dateData) write({ type: `insert`, value: row }) + commit() + markReady() + }, + }, + }) + } + + async function queryDateBothPaths(where: BasicExpression) { + const indexed = makeDateCollection() + await indexed.stateWhenReady() + indexed.createIndex((row) => row.createdAt) + const fullScan = makeDateCollection() + await fullScan.stateWhenReady() + + const indexedIds = indexed + .currentStateAsChanges({ where })! + .map((c) => c.value.id) + .sort() + const fullScanIds = fullScan + .currentStateAsChanges({ where })! + .map((c) => c.value.id) + .sort() + + expect(indexedIds).toEqual(fullScanIds) + return indexedIds + } + + it(`matches invalid-Date rows for equality on an invalid Date`, async () => { + expect( + await queryDateBothPaths( + eq(new PropRef([`createdAt`]), new Date(`not a date`)), + ), + ).toEqual([`invalid`]) + }) + + it(`matches an invalid-Date member of an IN list`, async () => { + expect( + await queryDateBothPaths( + inArray(new PropRef([`createdAt`]), [new Date(`not a date`), valid]), + ), + ).toEqual([`invalid`, `valid`]) + }) + + it(`treats an invalid Date as greater than valid Dates in a range query`, async () => { + // createdAt > 2022 matches the valid Date and the invalid Date (greatest) + expect( + await queryDateBothPaths( + gt(new PropRef([`createdAt`]), new Date(`2022-01-01`)), + ), + ).toEqual([`invalid`, `valid`]) + }) +}) diff --git a/packages/db/tests/query/compiler/evaluators.test.ts b/packages/db/tests/query/compiler/evaluators.test.ts index 69969de18a..4c5acb78c1 100644 --- a/packages/db/tests/query/compiler/evaluators.test.ts +++ b/packages/db/tests/query/compiler/evaluators.test.ts @@ -730,6 +730,87 @@ describe(`evaluators`, () => { expect(compiled({})).toBe(null) }) }) + + describe(`NaN (PostgreSQL float semantics)`, () => { + // Following PostgreSQL, NaN is equal to itself and greater than every + // other (non-null) value, so it has a well-defined order. + it(`treats NaN as equal to NaN`, () => { + const func = new Func(`eq`, [new Value(NaN), new Value(NaN)]) + expect(compileExpression(func)({})).toBe(true) + }) + + it(`treats NaN as not equal to a number`, () => { + const func = new Func(`eq`, [new Value(NaN), new Value(5)]) + expect(compileExpression(func)({})).toBe(false) + }) + + it(`still returns UNKNOWN when comparing NaN with null`, () => { + const func = new Func(`eq`, [new Value(NaN), new Value(null)]) + expect(compileExpression(func)({})).toBe(null) + }) + + it(`treats NaN as greater than every number`, () => { + expect( + compileExpression(new Func(`gt`, [new Value(NaN), new Value(5)]))( + {}, + ), + ).toBe(true) + expect( + compileExpression(new Func(`gt`, [new Value(5), new Value(NaN)]))( + {}, + ), + ).toBe(false) + expect( + compileExpression( + new Func(`gt`, [new Value(NaN), new Value(NaN)]), + )({}), + ).toBe(false) + }) + + it(`orders NaN with gte/lt/lte consistently`, () => { + // NaN >= anything (including NaN); nothing finite >= NaN + expect( + compileExpression( + new Func(`gte`, [new Value(NaN), new Value(5)]), + )({}), + ).toBe(true) + expect( + compileExpression( + new Func(`gte`, [new Value(NaN), new Value(NaN)]), + )({}), + ).toBe(true) + // NaN < nothing; a finite value < NaN + expect( + compileExpression(new Func(`lt`, [new Value(NaN), new Value(5)]))( + {}, + ), + ).toBe(false) + expect( + compileExpression(new Func(`lt`, [new Value(5), new Value(NaN)]))( + {}, + ), + ).toBe(true) + // NaN <= NaN; a finite value <= NaN + expect( + compileExpression( + new Func(`lte`, [new Value(NaN), new Value(NaN)]), + )({}), + ).toBe(true) + expect( + compileExpression( + new Func(`lte`, [new Value(5), new Value(NaN)]), + )({}), + ).toBe(true) + }) + + it(`matches NaN inside an IN list`, () => { + const func = new Func(`in`, [ + new Value(NaN), + new Value([NaN, 1, 2]), + ]) + expect(compileExpression(func)({})).toBe(true) + }) + }) }) describe(`boolean operators`, () => { From c7f07962e3af3a2f7fb70aea9b685584931f0f7c Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Fri, 26 Jun 2026 12:07:55 +0200 Subject: [PATCH 20/21] chore: mark NaN-semantics changeset as patch (no minor before 1.0) Co-Authored-By: Claude Opus 4.8 (1M context) --- .changeset/nan-postgres-semantics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/nan-postgres-semantics.md b/.changeset/nan-postgres-semantics.md index 63d7d1b2b9..028a372523 100644 --- a/.changeset/nan-postgres-semantics.md +++ b/.changeset/nan-postgres-semantics.md @@ -1,5 +1,5 @@ --- -'@tanstack/db': minor +'@tanstack/db': patch --- Adopt PostgreSQL float semantics for `NaN` in `where` clauses and ordering. From a155f99f5d0ae1a5938c477996bd53cfe8e95f32 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Fri, 26 Jun 2026 17:07:57 +0200 Subject: [PATCH 21/21] test: fold nan-semantics tests into existing well-suited test files Move the unique NaN/invalid-Date coverage (lt on NaN, invalid-Date eq/IN/range index parity) into collection-indexes.test.ts alongside the existing NaN index tests, and drop the standalone nan-semantics.test.ts whose other cases were already covered by collection-indexes, deterministic-ordering and evaluators tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/db/tests/collection-indexes.test.ts | 107 ++++++++++ packages/db/tests/nan-semantics.test.ts | 195 ------------------- 2 files changed, 107 insertions(+), 195 deletions(-) delete mode 100644 packages/db/tests/nan-semantics.test.ts diff --git a/packages/db/tests/collection-indexes.test.ts b/packages/db/tests/collection-indexes.test.ts index 3a136fa96c..bd5f4868c7 100644 --- a/packages/db/tests/collection-indexes.test.ts +++ b/packages/db/tests/collection-indexes.test.ts @@ -1617,6 +1617,113 @@ describe(`Collection Indexes`, () => { }) }) }) + + it(`should exclude NaN from a less-than range query`, async () => { + // Under PostgreSQL float semantics NaN is the greatest value, so + // `score < 4` matches the rows with scores 1 and 3 but never the + // NaN-valued row. + const nanCollection = createCollection< + { id: string; score: number }, + string + >({ + getKey: (row) => row.id, + startSync: true, + autoIndex: `off`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ type: `insert`, value: { id: `nan`, score: NaN } }) + write({ type: `insert`, value: { id: `one`, score: 1 } }) + write({ type: `insert`, value: { id: `three`, score: 3 } }) + write({ type: `insert`, value: { id: `five`, score: 5 } }) + write({ type: `insert`, value: { id: `seven`, score: 7 } }) + commit() + markReady() + }, + }, + }) + await nanCollection.stateWhenReady() + nanCollection.createIndex((row) => row.score) + + const result = nanCollection.currentStateAsChanges({ + where: lt(new PropRef([`score`]), 4), + })! + + const ids = result.map((r) => r.value.id).sort() + expect(ids).toEqual([`one`, `three`]) + }) + + // Invalid Dates have a NaN timestamp, so they follow the same PostgreSQL + // float semantics as NaN: equal to one another and greater than every valid + // Date. The index-served and full-scan results must agree. + const makeInvalidDateCollection = async () => { + const dateCollection = createCollection< + { id: string; createdAt: Date }, + string + >({ + getKey: (row) => row.id, + startSync: true, + autoIndex: `off`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ + type: `insert`, + value: { id: `invalid`, createdAt: new Date(`not a date`) }, + }) + write({ + type: `insert`, + value: { id: `valid`, createdAt: new Date(`2023-01-01`) }, + }) + commit() + markReady() + }, + }, + }) + await dateCollection.stateWhenReady() + dateCollection.createIndex((row) => row.createdAt) + return dateCollection + } + + it(`should match an invalid-Date row for an equality on an invalid Date`, async () => { + const dateCollection = await makeInvalidDateCollection() + + const result = dateCollection.currentStateAsChanges({ + where: eq(new PropRef([`createdAt`]), new Date(`not a date`)), + })! + + const ids = result.map((r) => r.value.id).sort() + expect(ids).toEqual([`invalid`]) + }) + + it(`should match an invalid-Date member of an IN list`, async () => { + const dateCollection = await makeInvalidDateCollection() + + const result = dateCollection.currentStateAsChanges({ + where: inArray(new PropRef([`createdAt`]), [ + new Date(`not a date`), + new Date(`2023-01-01`), + ]), + })! + + const ids = result.map((r) => r.value.id).sort() + expect(ids).toEqual([`invalid`, `valid`]) + }) + + it(`should treat an invalid Date as greater than valid Dates in a range query`, async () => { + // `createdAt > 2022` matches the valid Date and the invalid Date (which + // is the greatest value under PostgreSQL float semantics). + const dateCollection = await makeInvalidDateCollection() + + const result = dateCollection.currentStateAsChanges({ + where: gt(new PropRef([`createdAt`]), new Date(`2022-01-01`)), + })! + + const ids = result.map((r) => r.value.id).sort() + expect(ids).toEqual([`invalid`, `valid`]) + }) }) describe(`Index Usage Verification`, () => { diff --git a/packages/db/tests/nan-semantics.test.ts b/packages/db/tests/nan-semantics.test.ts deleted file mode 100644 index 6acca7fe3d..0000000000 --- a/packages/db/tests/nan-semantics.test.ts +++ /dev/null @@ -1,195 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { createCollection } from '../src/collection/index.js' -import { eq, gt, inArray, lt } from '../src/query/builder/functions' -import { PropRef } from '../src/query/ir' -import { BTreeIndex } from '../src/indexes/btree-index.js' -import type { BasicExpression } from '../src/query/ir' - -/** - * The query engine follows PostgreSQL float semantics for `NaN`: it is equal to - * itself and greater than every other (non-null) value, so it has a - * well-defined order and behaves like a normal value in `where` clauses and - * `orderBy`. These tests also assert that querying with an index produces the - * same result as a full scan. - */ -interface Row { - id: string - score: number -} - -const data: Array = [ - { id: `nan`, score: NaN }, - { id: `one`, score: 1 }, - { id: `three`, score: 3 }, - { id: `five`, score: 5 }, - { id: `seven`, score: 7 }, -] - -function makeCollection() { - const collection = createCollection({ - getKey: (row) => row.id, - startSync: true, - autoIndex: `off`, - defaultIndexType: BTreeIndex, - sync: { - sync: ({ begin, write, commit, markReady }) => { - begin() - for (const row of data) write({ type: `insert`, value: row }) - commit() - markReady() - }, - }, - }) - return collection -} - -async function queryBothPaths(where: BasicExpression) { - const indexed = makeCollection() - await indexed.stateWhenReady() - indexed.createIndex((row) => row.score) - const fullScan = makeCollection() - await fullScan.stateWhenReady() - - const indexedIds = indexed - .currentStateAsChanges({ where })! - .map((c) => c.value.id) - .sort() - const fullScanIds = fullScan - .currentStateAsChanges({ where })! - .map((c) => c.value.id) - .sort() - - // The chosen execution strategy must not change the result - expect(indexedIds).toEqual(fullScanIds) - return indexedIds -} - -describe(`NaN query semantics (PostgreSQL float semantics)`, () => { - it(`matches NaN rows for equality on NaN`, async () => { - expect(await queryBothPaths(eq(new PropRef([`score`]), NaN))).toEqual([ - `nan`, - ]) - }) - - it(`treats NaN as greater than every number in a range query`, async () => { - // score > 2 matches 3, 5, 7 and NaN (NaN is greatest) - expect(await queryBothPaths(gt(new PropRef([`score`]), 2))).toEqual([ - `five`, - `nan`, - `seven`, - `three`, - ]) - }) - - it(`excludes NaN from a less-than range query`, async () => { - // score < 4 matches 1 and 3, but not NaN (NaN is greatest) - expect(await queryBothPaths(lt(new PropRef([`score`]), 4))).toEqual([ - `one`, - `three`, - ]) - }) - - it(`matches a NaN member of an IN list`, async () => { - expect( - await queryBothPaths(inArray(new PropRef([`score`]), [NaN, 3])), - ).toEqual([`nan`, `three`]) - }) - - it(`orders NaN last when sorting ascending`, async () => { - const collection = makeCollection() - await collection.stateWhenReady() - - const ordered = collection - .currentStateAsChanges({ - orderBy: [ - { - expression: new PropRef([`score`]), - compareOptions: { direction: `asc`, nulls: `first` }, - }, - ], - })! - .map((c) => c.value.id) - - expect(ordered).toEqual([`one`, `three`, `five`, `seven`, `nan`]) - }) -}) - -/** - * Invalid Dates have a `NaN` timestamp, so they follow the same PostgreSQL - * float semantics as `NaN`: equal to one another and greater than every valid - * Date. The indexed and full-scan paths must agree. - */ -describe(`invalid Date query semantics (PostgreSQL float semantics)`, () => { - interface DateRow { - id: string - createdAt: Date - } - - const valid = new Date(`2023-01-01`) - const dateData: Array = [ - { id: `invalid`, createdAt: new Date(`not a date`) }, - { id: `valid`, createdAt: valid }, - ] - - function makeDateCollection() { - return createCollection({ - getKey: (row) => row.id, - startSync: true, - autoIndex: `off`, - defaultIndexType: BTreeIndex, - sync: { - sync: ({ begin, write, commit, markReady }) => { - begin() - for (const row of dateData) write({ type: `insert`, value: row }) - commit() - markReady() - }, - }, - }) - } - - async function queryDateBothPaths(where: BasicExpression) { - const indexed = makeDateCollection() - await indexed.stateWhenReady() - indexed.createIndex((row) => row.createdAt) - const fullScan = makeDateCollection() - await fullScan.stateWhenReady() - - const indexedIds = indexed - .currentStateAsChanges({ where })! - .map((c) => c.value.id) - .sort() - const fullScanIds = fullScan - .currentStateAsChanges({ where })! - .map((c) => c.value.id) - .sort() - - expect(indexedIds).toEqual(fullScanIds) - return indexedIds - } - - it(`matches invalid-Date rows for equality on an invalid Date`, async () => { - expect( - await queryDateBothPaths( - eq(new PropRef([`createdAt`]), new Date(`not a date`)), - ), - ).toEqual([`invalid`]) - }) - - it(`matches an invalid-Date member of an IN list`, async () => { - expect( - await queryDateBothPaths( - inArray(new PropRef([`createdAt`]), [new Date(`not a date`), valid]), - ), - ).toEqual([`invalid`, `valid`]) - }) - - it(`treats an invalid Date as greater than valid Dates in a range query`, async () => { - // createdAt > 2022 matches the valid Date and the invalid Date (greatest) - expect( - await queryDateBothPaths( - gt(new PropRef([`createdAt`]), new Date(`2022-01-01`)), - ), - ).toEqual([`invalid`, `valid`]) - }) -})