diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index f7128fb41e..b9305be18d 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -241,7 +241,7 @@ comment and the current API/architecture contract before extending its model. | Surface | Primary executable owners | Independent judgment and important limit | | -------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Ordered relations and BTree | [top-K relation oracle](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/operators/topk-relation-oracle.test.ts), [fractional-index window moves](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/operators/topKWithFractionalIndex-oracle.test.ts), [BTree/Map](https://github.com/TanStack/db/blob/main/packages/db/tests/btree-map-oracle.test.ts), [incrementalization laws](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/incrementalization-law-oracle.property.test.ts) | Independent ordered relations and cumulative signed output. Top-K consolidation compares same-key values without hashing, including cyclic replacements and fresh transient cancellation. The fractional-index test checks an empty output message for a window move after all rows retract; it does not prove downstream lazy acquisition. Other hash-based operators retain hashing's declared domain. Algebra does not specify client readiness. The BTree/Map oracle checks point operations, neighbor pairs, and full, partial, boundary, exclusive-high, and empty range scans for integer keys and node sizes 4–8. It does not check balance or space, and it assumes consistent comparators ([code-weight BTree review](oracle-reviews/code-weight-btree-trim.md)). | +| Ordered relations and BTree | [top-K relation oracle](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/operators/topk-relation-oracle.test.ts), [fractional-index window moves](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/operators/topKWithFractionalIndex-oracle.test.ts), [BTree/Map](https://github.com/TanStack/db/blob/main/packages/db/tests/btree-map-oracle.test.ts), [incrementalization laws](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/incrementalization-law-oracle.property.test.ts) | Independent ordered relations and cumulative signed output. Top-K consolidation compares same-key values without hashing, including cyclic replacements and fresh transient cancellation. The fractional-index test checks an empty output message for a window move after all rows retract; it does not prove downstream lazy acquisition. Other hash-based operators retain hashing's declared domain. Algebra does not specify client readiness. The BTree/Map oracle checks point operations, neighbor pairs, and full, partial, boundary, exclusive-high, and empty range scans for integer keys and node sizes 4–8. It does not check balance or space, and it assumes consistent comparators ([code-weight BTree review](oracle-reviews/code-weight-btree-trim.md)). The generated groupBy law compares JSON values; a pinned contract test in `packages/db-ivm/tests/operators/groupBy.test.ts` records that after a delete, `min` and `max` return a value hash-equal to a remaining member, not necessarily the remaining instance (`-0` for `0`, an equal Date), which is why the query compiler keeps its own exact-value keys. It checks both insertion orders, reads the group's row by integrating output multiplicities, deletes the minimum and the maximum, and shows that an aggregate input that tells the rows apart returns the remaining instance, for -0/0 in both directions and for both `min` and `max`. The contract holds while every member has positive multiplicity. It pins the current behavior separately: the delete's retraction is the last record a shared hash entry gets, so the deleted instance comes back in either insertion order, for -0/0 in either direction and for an equal Date. For -0/0 and equal Dates the contract assertions only show hash equality, which is all the contract promises. Open: the incrementalization law's value policy compares JSON and clones by `structuredClone`, so no generated history sees -0 or Date identity; a value-identity policy with -0/0 and equal-time Dates would generalize this pinned contract. | | Includes and publication | [cross-formulation](https://github.com/TanStack/db/blob/main/packages/db/tests/query/includes-cross-formulation-oracle.property.test.ts), [temporal](https://github.com/TanStack/db/blob/main/packages/db/tests/query/includes-temporal-oracle.test.ts), [Collection includes](https://github.com/TanStack/db/blob/main/packages/db/tests/query/includes-collection-oracle.property.test.ts), [architecture and complete suite map](https://github.com/TanStack/db/blob/main/packages/db/src/query/live/ARCHITECTURE.md#executable-contracts) | Per-parent/flat-join/partition relations, callback-time rows, nested values, and route histories. Fragmented lazy-demand consolidation covers delta growth and successful, rejected/retried, and obsolete replacements at the Collection request boundary; it observes exact key unions and abort checkpoints. A compiled-includes fixture makes adapter unload evict owned child rows and proves a rejected union cannot remove established public child rows. Retry after that terminal live-query failure remains covered at the acquisition boundary; no automatic in-place recovery is claimed. Observe raw promised order; fresh queries do not establish continuous publication safety. | | Collection lifecycle | [mutation startup](https://github.com/TanStack/db/blob/main/packages/db/tests/collection-mutation-startup-oracle.test.ts), [change-event history](https://github.com/TanStack/db/blob/main/packages/db/tests/change-event-history-oracle.test.ts), [sync publication reentrancy](https://github.com/TanStack/db/blob/main/packages/db/tests/collection-sync-reentrancy-oracle.test.ts), [history](https://github.com/TanStack/db/blob/main/packages/db/tests/collection-subscription-lifecycle-history-oracle.property.test.ts), [publication](https://github.com/TanStack/db/blob/main/packages/db/tests/collection-subscription-lifecycle-publication-oracle.property.test.ts), [replay](https://github.com/TanStack/db/blob/main/packages/db/tests/collection-subscription-replay-oracle.property.test.ts), [effect disposal](https://github.com/TanStack/db/blob/main/packages/db/tests/effect-disposal-oracle.test.ts), [PR #1902 review](oracle-reviews/pr-1902-change-event-history.md), [batch-order decision review](oracle-reviews/change-event-batch-order.md), [absent-key insert review](oracle-reviews/2026-10-02-insert-for-absent-key.md) | Core Collection `insert`/`update`/`delete` admission while `startSync:false` is idle; complete public-row/change-message agreement across bounded one- and two-key histories plus replayable four-key campaigns, where every message must be valid for a strict consumer (insert only for an absent key, update or delete only for a present one; [absent-key insert review](oracle-reviews/2026-10-02-insert-for-absent-key.md)); a two-key deferred sync history checks each key's causal insert/update/delete trace while allowing any cross-key interleaving. Reversing all deferred messages fails that check; reversing each sync transaction's distinct-key messages passes. Queued duplicate admission and cancellation, ownership and phase histories, exact caller/error/publication evidence, and late completion and restart have separate witnesses. Generated deferred histories with cancellation or reentrant callbacks need a witness in the change-event history owner before claiming general batch-shape coverage. Query write utilities and effect self-dependent disposal remain separate contracts. During a truncate, a ready callback can write to any key after a prior optimistic request. Each subscriber, with or without initial state or a filter, must receive valid messages after the Collection is ready and end at the Collection's rows. A throwing ready callback during sync entry surfaces once the sync function returns, whether it calls markReady or commits a truncate ([round 3 follow-ups](oracle-reviews/2026-10-05-round-3-followups.md)). Callback rollbacks, non-optimistic callback writes, and several writing callbacks remain open ([ready-callback truncate review](oracle-reviews/2026-10-05-truncate-ready-callback-delete.md)). | | Paced mutations | [virtual-clock oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/paced-mutations-oracle.test.ts) | `createPacedMutations` with queue front/back options, bounded waiting capacity at zero and one, cleanup draining in FIFO/LIFO order after a held write, cleanup timing at the configured wait for immediate writes, admitted writes after separate Collection cleanup, debounce and throttle schedules with explicit and omitted options, and first-leading throttle execution at epoch zero. Finite histories compare optimistic rows, returned transaction identity and receipt outcome, execution order and virtual-clock time. Held writes verify queue serialization. Leading-only throttle and debounce witnesses reject skipped calls immediately with their named dropped-call errors, including omitted edges, both edges disabled, and same-row rollback while a prior write is held. Capacity witnesses cover overflow rejection, distinct-key optimistic rollback, same-key admitted-write survival, and in-flight versus waiting admission. Cleanup witnesses check repeated queue cleanup, eventual admitted receipt settlement, direct queue strategy rejection of new callbacks, public post-cleanup rollback with `QueueDisposedError`, pending debounce transactions that wait until the last call's quiet edge after separate Collection cleanup, and a trailing throttle timer that runs at its regular edge after separate Collection cleanup. Deferred custom queue and batch callbacks check compatibility with `void` and `false` execute results, respectively. Frozen options verify factory non-mutation; a mutable option witness checks that debounce uses its construction-time trailing setting. Throwing `onMutate` histories check leading-edge reuse before any row change, failed-row restoration when a pending transaction already has admitted work, eligible leading writes behind a held handler, and throttle pending merges at the throw, start, receipt, and later-timer cuts. New debounce/throttle admission after cleanup, manual commit of a strategy-owned transaction, longer throwing/reentrant bursts, failed persistence, broader capacities and schedules remain outside this owner. | diff --git a/packages/db-ivm/tests/operators/groupBy.test.ts b/packages/db-ivm/tests/operators/groupBy.test.ts index 63067bc1b4..037ab7f8cf 100644 --- a/packages/db-ivm/tests/operators/groupBy.test.ts +++ b/packages/db-ivm/tests/operators/groupBy.test.ts @@ -13,6 +13,7 @@ import { } from '../../src/operators/groupBy.js' import { output } from '../../src/operators/index.js' import { serializeValue } from '../../src/utils.js' +import { hash } from '../../src/hashing/index.js' describe(`Operators`, () => { describe(`GroupBy operation`, () => { @@ -624,6 +625,155 @@ describe(`Operators`, () => { expect(latestMessage.getInner()).toEqual(expectedResult) }) + // Contract for callers, not a replay witness: the generated groupBy law in + // incrementalization-law-oracle.property.test.ts compares JSON values and + // cannot see -0 or Date identity. groupBy's reduce keeps a group's + // contributions in an Index keyed by the hash of each row's record of + // pre-mapped aggregate inputs. Rows whose records hash equal share one + // entry, which sums their multiplicities and keeps the instance it was + // last given, a retraction's included. The hash + // treats -0 as 0 and equal Dates as one value. So after a delete, min and + // max return a value hash-equal to a remaining member's, not necessarily + // that member's instance. This holds while every member has positive + // multiplicity: min and max do not look at multiplicities. An aggregate + // input that tells the rows apart keeps their records apart. The query + // compiler adds one for min and max: an exact-value key that separates -0 + // from 0 and gives each object its row key. A distinct remaining value is + // returned as is. For -0/0 and equal Dates, hash equality is all the + // contract promises, so those cases only show the result stays in the + // remaining value's class; the distinct and per-row cases are the ones a + // wrong value fails. + // + // The group's row after the delete is integrated from the output's + // multiplicities by hash, keeping the instance of the latest insertion: a + // retraction is a new object. Whether a hash-equal row is published again + // is not part of this. + const extremes = ( + deletedValue: number | Date, + keptValue: number | Date, + options: { deletedFirst?: boolean; perRowInput?: boolean } = {}, + ) => { + const { deletedFirst = true, perRowInput = false } = options + const graph = new D2() + const input = graph.newInput<{ + g: string + id: number + v: number | Date + }>() + const rows = new Map() + input.pipe( + groupBy((row) => ({ g: row.g }), { + lo: min((row) => row.v), + hi: max((row) => row.v), + ...(perRowInput ? { id: max((row) => row.id) } : {}), + }), + output((message) => { + for (const [[, row], multiplicity] of message.getInner()) { + const entry = rows.get(hash(row)) ?? { row, count: 0 } + entry.count += multiplicity + if (multiplicity > 0) entry.row = row + rows.set(hash(row), entry) + } + }), + ) + graph.finalize() + const deleted = { g: `x`, id: 1, v: deletedValue } + const kept = { g: `x`, id: 2, v: keptValue } + input.sendData( + new MultiSet( + deletedFirst + ? [ + [deleted, 1], + [kept, 1], + ] + : [ + [kept, 1], + [deleted, 1], + ], + ), + ) + graph.run() + input.sendData(new MultiSet([[deleted, -1]])) + graph.run() + const present = [...rows.values()].filter((entry) => entry.count !== 0) + expect( + present.map((entry) => entry.count), + `one row for the group`, + ).toEqual([1]) + return present[0]!.row as { lo: unknown; hi: unknown } + } + + test(`min and max after a delete return a value hash-equal to the remaining one`, () => { + for (const deletedFirst of [true, false]) { + for (const [deleted, kept] of [ + [-0, 0], + [0, -0], + ] as const) { + const zero = extremes(deleted, kept, { deletedFirst }) + expect(hash(zero.lo)).toBe(hash(kept)) + expect(hash(zero.hi)).toBe(hash(kept)) + } + const keptDate = new Date(5) + const date = extremes(new Date(5), keptDate, { deletedFirst }) + expect(hash(date.lo)).toBe(hash(keptDate)) + expect(hash(date.hi)).toBe(hash(keptDate)) + + // A distinct remaining value is returned as is, whether the delete + // removes the minimum or the maximum. + expect(extremes(1, 2, { deletedFirst })).toMatchObject({ lo: 2, hi: 2 }) + expect(extremes(2, 1, { deletedFirst })).toMatchObject({ lo: 1, hi: 1 }) + const distinct = new Date(6) + const distinctDates = extremes(new Date(5), distinct, { deletedFirst }) + expect(distinctDates.lo).toBe(distinct) + expect(distinctDates.hi).toBe(distinct) + const earlier = new Date(5) + const earlierDates = extremes(new Date(6), earlier, { deletedFirst }) + expect(earlierDates.lo).toBe(earlier) + expect(earlierDates.hi).toBe(earlier) + + // An input that differs per row keeps the records apart, so the + // remaining instance comes back. + const exactDate = new Date(5) + const exact = extremes(new Date(5), exactDate, { + deletedFirst, + perRowInput: true, + }) + expect(exact.lo).toBe(exactDate) + expect(exact.hi).toBe(exactDate) + for (const [deleted, kept] of [ + [-0, 0], + [0, -0], + ] as const) { + const zero = extremes(deleted, kept, { + deletedFirst, + perRowInput: true, + }) + expect(Object.is(zero.lo, kept)).toBe(true) + expect(Object.is(zero.hi, kept)).toBe(true) + } + } + }) + + test(`min and max can return the deleted instance of a hash-equal value`, () => { + // Current behavior, pinned so a db-ivm change alerts the compiler owner. + // The delete's retraction is the last record the shared entry gets, so + // its instance, the deleted value, is the one that comes back, in either + // insertion order. + for (const deletedFirst of [true, false]) { + for (const deleted of [-0, 0]) { + const zero = extremes(deleted, deleted === 0 ? -0 : 0, { + deletedFirst, + }) + expect(Object.is(zero.lo, deleted)).toBe(true) + expect(Object.is(zero.hi, deleted)).toBe(true) + } + const deletedDate = new Date(5) + const date = extremes(deletedDate, new Date(5), { deletedFirst }) + expect(date.lo).toBe(deletedDate) + expect(date.hi).toBe(deletedDate) + } + }) + // These are readable replay witnesses. The generated groupBy law lives in // incrementalization-law-oracle.property.test.ts. test(`min and max reduce keep 0, 0n, and empty string as extremes`, () => { diff --git a/packages/db/src/query/compiler/group-by.ts b/packages/db/src/query/compiler/group-by.ts index 1d672acd8f..92830888eb 100644 --- a/packages/db/src/query/compiler/group-by.ts +++ b/packages/db/src/query/compiler/group-by.ts @@ -430,15 +430,16 @@ export function processGroupBy( } } - // D2 consolidates contributions whose hashes match, and the hash treats -0 - // as 0 and equal Dates as one value. A min or max returns one of its - // inputs, so each contribution carries the exact key of the value it - // compares, as a group value does. A primitive's key is its exact value, so - // equal primitives still consolidate. An object's key holds its row key, so - // a retraction cancels its insert even when the argument builds a new - // instance each time, and a merged contribution never keeps a deleted - // row's instance. A sum, avg, or count only adds its coerced numbers, so - // merging equal inputs cannot change its result. + // D2 consolidates contributions whose hashes match, and the hash treats -0 as 0 + // and equal Dates as one value. Today it keeps the instance it was last given; + // db-ivm's groupBy tests pin that as current behavior, not as a promise. A min + // or max returns one of its inputs, so each contribution carries the exact key + // of the value it compares, as a group value does. A primitive's key is its + // exact value, so equal primitives still consolidate. An object's key holds its + // row key, so a retraction cancels its insert even when the argument builds a + // new instance each time, and a merged contribution never keeps a deleted row's + // instance. A sum, avg, or count only adds its coerced numbers, so merging + // equal inputs cannot change its result. const exactInputs: Array<(entry: [string, NamespacedRow]) => unknown> = [] const addAggregate = (alias: string, aggExpr: Aggregate) => { aggregates[alias] = getAggregateFunction(aggExpr)