diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b364f0e8aca..e0a843f4429 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -729,6 +729,9 @@ jobs: with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + # check-editions compares against the merge base, so it needs real history. + fetch-depth: 0 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -738,6 +741,16 @@ jobs: run: | cargo run --profile ci -p xtask -- generate-fbs cargo run --profile ci -p xtask -- generate-proto + - name: "regenerate the edition records" + run: | + cargo run --profile ci -p xtask -- generate-editions + - name: "check frozen edition records never change" + # Independent of the regeneration above: a stale record must not mask a frozen one + # being edited, nor the other way round. + if: "!cancelled()" + run: | + BASE="${{ github.event.pull_request.base.sha || 'HEAD^' }}" + cargo run --profile ci -p xtask -- check-editions --base "$BASE" - name: "regenerate FFI header file" run: | cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi diff --git a/Cargo.lock b/Cargo.lock index 2448cb9aff8..3e8ed089b25 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4507,6 +4507,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "git2" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" +dependencies = [ + "bitflags 2.13.1", + "libc", + "libgit2-sys", + "log", +] + [[package]] name = "glob" version = "0.3.4" @@ -6200,6 +6212,18 @@ dependencies = [ "cc", ] +[[package]] +name = "libgit2-sys" +version = "0.18.7+1.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23c7391e4b9f4ffab1a624223cc1d7385ff9a678f490768add717de7ea2f4d89" +dependencies = [ + "cc", + "libc", + "libz-sys", + "pkg-config", +] + [[package]] name = "libloading" version = "0.8.9" @@ -6285,6 +6309,18 @@ dependencies = [ "escape8259", ] +[[package]] +name = "libz-sys" +version = "1.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "line-clipping" version = "0.3.8" @@ -10413,6 +10449,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -10455,6 +10497,7 @@ dependencies = [ "vortex-layout", "vortex-mask", "vortex-metrics", + "vortex-parquet-variant", "vortex-pco", "vortex-proto", "vortex-runend", @@ -11585,6 +11628,7 @@ dependencies = [ "vortex-btrblocks", "vortex-buffer", "vortex-compressor", + "vortex-edition", "vortex-error", "vortex-mask", "vortex-session", @@ -11684,6 +11728,7 @@ dependencies = [ "rstest", "vortex-array", "vortex-buffer", + "vortex-edition", "vortex-error", "vortex-mask", "vortex-session", @@ -12199,7 +12244,14 @@ version = "0.1.0" dependencies = [ "anyhow", "clap", + "git2", "prost-build", + "toml", + "vortex-edition", + "vortex-json", + "vortex-spatial", + "vortex-tensor", + "vortex-zstd", "xshell", ] diff --git a/Cargo.toml b/Cargo.toml index ae164db13c2..37f66df1443 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -171,6 +171,7 @@ geo-types = "0.7.19" geoarrow = "0.8.0" geoarrow-cast = "0.8.0" get_dir = "0.5.0" +git2 = { version = "0.21", default-features = false } glob = "0.3.2" goldenfile = "1" half = { version = "2.7.1", features = ["std", "num-traits"] } @@ -276,6 +277,7 @@ thiserror = "2.0.3" tokio = { version = "1.52" } tokio-stream = "0.1.17" tokio-util = "0.7.17" +toml = "0.9" tpchgen = "3.0.0" tpchgen-arrow = "3.0.0" tracing = { version = "0.1.41", default-features = false } diff --git a/benchmarks/compress-bench/Cargo.toml b/benchmarks/compress-bench/Cargo.toml index b044dfe1263..6fb0f24df47 100644 --- a/benchmarks/compress-bench/Cargo.toml +++ b/benchmarks/compress-bench/Cargo.toml @@ -41,7 +41,11 @@ vortex-cuda = { workspace = true, optional = true } [features] cuda = ["dep:tempfile", "dep:vortex-cuda"] lance = ["dep:lance-bench"] -unstable_encodings = ["vortex/unstable_encodings", "vortex-cuda?/unstable_encodings"] +unstable_encodings = [ + "vortex/unstable_encodings", + "vortex-bench/unstable_encodings", + "vortex-cuda?/unstable_encodings", +] [[bin]] name = "compress-bench" diff --git a/benchmarks/compress-bench/src/gpu/vortex.rs b/benchmarks/compress-bench/src/gpu/vortex.rs index 8c09ae2f44e..be1ab61868f 100644 --- a/benchmarks/compress-bench/src/gpu/vortex.rs +++ b/benchmarks/compress-bench/src/gpu/vortex.rs @@ -32,6 +32,7 @@ use vortex::layout::scan::split_by::SplitBy; use vortex_arrow::ArrowSessionExt; use vortex_bench::Format; use vortex_bench::SESSION; +use vortex_bench::benchmark_write_options; use vortex_bench::compress::Compressor; use vortex_bench::conversions::parquet_to_vortex_chunks_with_batch_size; use vortex_cuda::CanonicalCudaExt; @@ -95,8 +96,7 @@ impl Compressor for GpuVortexCompressor { .only_cuda_compatible() .build(), ))); - SESSION - .write_options() + benchmark_write_options(SESSION.write_options()) .with_strategy(strategy) .write(&mut output, uncompressed.into_array().to_array_stream()) .await?; diff --git a/benchmarks/compress-bench/src/vortex.rs b/benchmarks/compress-bench/src/vortex.rs index 20b4b9f1402..a030a54b903 100644 --- a/benchmarks/compress-bench/src/vortex.rs +++ b/benchmarks/compress-bench/src/vortex.rs @@ -21,6 +21,7 @@ use vortex::file::WriteOptionsSessionExt; use vortex_arrow::ArrowSessionExt; use vortex_bench::Format; use vortex_bench::SESSION; +use vortex_bench::benchmark_write_options; use vortex_bench::compress::Compressor; use vortex_bench::compress::read_projection; use vortex_bench::conversions::parquet_to_vortex_chunks; @@ -41,8 +42,7 @@ impl Compressor for VortexCompressor { let mut buf = Vec::new(); let start = Instant::now(); let mut cursor = Cursor::new(&mut buf); - SESSION - .write_options() + benchmark_write_options(SESSION.write_options()) .write(&mut cursor, uncompressed.into_array().to_array_stream()) .await?; let elapsed = start.elapsed(); @@ -55,8 +55,7 @@ impl Compressor for VortexCompressor { let uncompressed = parquet_to_vortex_chunks(parquet_path.to_path_buf()).await?; let mut buf = Vec::new(); let mut cursor = Cursor::new(&mut buf); - SESSION - .write_options() + benchmark_write_options(SESSION.write_options()) .write(&mut cursor, uncompressed.into_array().to_array_stream()) .await?; diff --git a/benchmarks/datafusion-bench/Cargo.toml b/benchmarks/datafusion-bench/Cargo.toml index 306669754d8..cc62efebf1b 100644 --- a/benchmarks/datafusion-bench/Cargo.toml +++ b/benchmarks/datafusion-bench/Cargo.toml @@ -58,7 +58,7 @@ custom-labels = { workspace = true } [features] cuda = ["dep:vortex-cuda"] -unstable_encodings = ["vortex/unstable_encodings"] +unstable_encodings = ["vortex/unstable_encodings", "vortex-bench/unstable_encodings"] [lints] workspace = true diff --git a/benchmarks/duckdb-bench/Cargo.toml b/benchmarks/duckdb-bench/Cargo.toml index 609f2fe2150..212282de7e6 100644 --- a/benchmarks/duckdb-bench/Cargo.toml +++ b/benchmarks/duckdb-bench/Cargo.toml @@ -37,7 +37,7 @@ vortex-duckdb = { workspace = true } [features] cuda = ["dep:vortex-cuda"] -unstable_encodings = ["vortex/unstable_encodings"] +unstable_encodings = ["vortex/unstable_encodings", "vortex-bench/unstable_encodings"] [lints] workspace = true diff --git a/benchmarks/string-bench/Cargo.toml b/benchmarks/string-bench/Cargo.toml index e2d3736b195..eae7d10f3aa 100644 --- a/benchmarks/string-bench/Cargo.toml +++ b/benchmarks/string-bench/Cargo.toml @@ -35,6 +35,7 @@ vortex-onpair = { workspace = true } [features] unstable_encodings = [ "vortex/unstable_encodings", + "vortex-bench/unstable_encodings", "vortex-btrblocks/unstable_encodings", ] diff --git a/benchmarks/string-bench/src/serialized.rs b/benchmarks/string-bench/src/serialized.rs index bb9f5fbe872..69232b95edb 100644 --- a/benchmarks/string-bench/src/serialized.rs +++ b/benchmarks/string-bench/src/serialized.rs @@ -38,6 +38,7 @@ use vortex::file::WriteStrategyBuilder; use vortex::layout::LayoutStrategy; use vortex::session::VortexSession; use vortex_bench::Format; +use vortex_bench::benchmark_write_options; use vortex_bench::measurements::CustomUnitMeasurement; use vortex_btrblocks::SchemeExt; use vortex_btrblocks::SchemeId; @@ -191,8 +192,7 @@ async fn write_serialized_file( let mut buf = Vec::new(); { let mut cursor = Cursor::new(&mut buf); - session - .write_options() + benchmark_write_options(session.write_options()) .with_strategy(Arc::clone(strategy)) .write(&mut cursor, input.to_array_stream()) .await?; diff --git a/docs/specs/editions.md b/docs/specs/editions.md index b3c6d721107..9984f4c559b 100644 --- a/docs/specs/editions.md +++ b/docs/specs/editions.md @@ -1,38 +1,37 @@ # Editions Vortex files contain several kinds of serialized **component**: array encodings, layout encodings, extension dtypes, and -aggregate functions. An **edition** is a named set of these components. It controls what a writer may put in a file and, -once frozen, identifies the earliest Vortex release that supports every component in the set. - -Each component consists of a kind, an ID, and the wire contract for its metadata and payload. The compatibility -guarantee applies to that serialized contract, not to the in-memory implementation that reads or writes it. +aggregate functions. An **edition** is a named set of their concrete wire IDs. It controls what a writer may put in a +file and, once frozen, identifies its origin library or project and minimum version: the earliest release of that origin +that recognizes every ID in the set. Editions belong to independently versioned families and are cumulative within a family. Each edition includes all -components from the preceding edition in that family, plus any newly added components. A writer selects at most one -edition from each family and may use the union of their components. For example, selecting `core2026.08.1` and -`preview2026.06.0` allows stable components released through August 2026 and preview components released through -June 2026. +components from the preceding edition in that family, plus any additions. A writer selects at most one edition from +each family and may use the union of their component IDs. For example, selecting `core2026.08.3` and +`tensor2026.04.0` allows the core components together with tensor arrays and dtypes. Every family names the origin +library or project whose release versions its editions use. The first frozen edition, `core2025.05.0`, contains the components that Vortex `0.36.0` could write. This marks the start of the Vortex file format's stability guarantee. Every Vortex release from `0.36.0` onward can read `core2025.05.0`, and later frozen `core` editions extend that guarantee to newer components. -When a writer selects only frozen editions, the highest of their minimum Vortex releases is the earliest release -guaranteed to read the resulting file. Draft editions have no minimum reader version; selecting one gives up this -guarantee for any draft components written to the file. +When a writer selects only frozen editions from one origin, the highest of their minimum library versions is the +earliest release guaranteed to read the resulting file. With multiple origins, the file requires the recorded minimum +version of each. Editions without minimum library versions are drafts and carry no guarantee about their future +compatibility. ## What an edition contains -An edition records every component by kind and ID. IDs are unique within a kind, but not across kinds: a layout named -`vortex.flat` and an array encoding with the same ID are distinct components. The writer therefore builds and enforces a -separate allowlist for each kind: +An edition records every component by kind and wire ID. IDs are unique within a kind, but not across kinds: a layout +named `vortex.flat` and an array encoding with the same ID are distinct components. The writer therefore builds and +enforces a separate allowlist for each kind: -| Kind | Written | Enforced at | +| Kind | What it identifies | Used at | |-------------|--------------------------------------------|------------------------------| -| `array` | every serialized array | array serialization context | +| `array` | a serialized array representation | array serialization | | `layout` | the footer's layout tree | layout serialization context | | `dtype` | extension dtypes nested in the file schema | file writer | -| `aggregate` | zone maps in zoned layouts | the layout writer context | +| `aggregate` | zone maps in zoned layouts | layout writer context | Writing a component that is absent from the selected editions fails the write. This rule applies to every kind, including aggregates. Although a zone map is only an optimization and could be dropped, doing so would silently change @@ -41,25 +40,27 @@ the writer's configured pruning behavior. Only aggregates that would actually be written are checked. If a column's dtype cannot support an aggregate, the writer omits it and there is no edition violation. -An empty allowlist permits no encodings. Collectively, the selected editions must declare every array encoding, layout -encoding, extension dtype, and aggregate function that the writer serializes. +An empty allowlist permits no components. Collectively, the selected editions must declare every serialized array ID, +layout encoding, extension dtype, and aggregate function that the writer writes. An array serializer may expose several +wire IDs for one in-memory encoding. The serializer chooses the representation, and the serialization context rejects +the write if the chosen wire ID is not declared by the selected editions. For example, `core2026.08.0` declares the aggregate functions that the default writer may store in zone maps: `min`, `max`, `bounded_min`, `bounded_max`, `nan_count`, and `null_count`. It does not declare `sum`, because the writer does not store sums in zone maps. File-level statistics use a fixed legacy field for sums rather than a serialized aggregate function ID, so this allowlist does not apply to them. -Optional Vortex modules enable their own edition families alongside `core`. Spatial support enables -`spatial2026.08.0`, for example, while JSON support enables `json2026.08.0`. +Optional Vortex modules enable their own edition families alongside `core`. Tensor support enables +`tensor2026.04.0`, for example, while Zstd buffer wrapping enables `zstd2026.02.0`. ## Resolving an unknown-component error An unknown-ID error means that the reader does not recognize a serialized component in the file. Find the component's kind and ID in the [registry](#edition-registry): -1. **It belongs to a frozen edition.** Upgrade to at least the minimum Vortex release listed for that edition. +1. **It belongs to a frozen edition.** Upgrade the edition's named origin to at least its minimum library version. 2. **It belongs to a draft edition.** No released reader is guaranteed to support it. Use a build that registers the - component or ask the file's producer which build to use. + component or ask the file's producer which build to use. 3. **It is not in the registry.** The file contains a custom, third-party, or experimental component outside the editions system. Ask the producer for its implementation and register it with the reader's session. @@ -69,9 +70,11 @@ zone-map pruning rather than causing the file to be rejected. ## Writing with an edition -By default, the Vortex facade targets the newest frozen `core` edition. New components may first appear in a draft -edition before joining a later frozen `core` edition. If serialization would use a component outside the selected -editions, the write fails immediately. +By default, the Vortex facade targets `core2026.08.3`. A new encoding or serialization feature that is +still evolving gets a new draft edition; later additions create later editions rather than changing an already +published feature set. Each feature advances through its own independently versioned family until it is ready to join +the shared `preview` family. Preview components remain opt-in until they are ready to join `core`. Components supplied +by an optional plugin belong to that plugin's family, such as `tensor`, `zstd`, `spatial`, or `json`. Edition configuration belongs to the writer's Vortex session. Registering an edition makes its declaration available to the session; enabling it allows the writer to use its components. Enabling another edition in the same family replaces @@ -80,49 +83,79 @@ the previous selection. You can change the default configuration to: - **Target an older `core` edition** when the file must remain readable by an older Vortex deployment. -- **Enable another family** to use components outside `core`. Vortex currently defines `preview`, `spatial`, and - `json` in addition to `core`. +- **Enable another family** to use components outside `core`. Vortex currently defines `preview`, `tensor`, `zstd`, + `spatial`, and `json` in addition to `core`. + +Sessions created without the Vortex facade must register and enable their editions before writing files. -Sessions created without the Vortex facade must register and enable their editions before writing files. The lower-level -`with_allow_encodings` policy can further restrict array encodings, but cannot permit an encoding excluded by the -selected editions. +For experimental or custom components that do not belong to an edition, the Rust writer exposes +`disable_editions()`. This disables every edition check for that write: every array representation registered in the +session is eligible for compression and serialization, while layouts, extension dtypes, and aggregate functions are +unrestricted. It does not register missing readers, so files written this way have no edition compatibility guarantee. + +Compression and edition compatibility are separate. Compressors produce current in-memory arrays and do not select a +wire ID. The writer maps each allowed serialized ID to its current in-memory encoding and restricts the default +BtrBlocks compressor to schemes producing those encodings. Custom compressors remain unrestricted, with serialization +providing the final compatibility boundary when edition enforcement is enabled. At that boundary, the array plugin +produces an ID, metadata, buffers, and children. The serialization context interns the returned ID and fails the write +if the selected editions do not permit it. A serializer may emit a historical ID when the value satisfies that ID's +frozen contract, but it does not inspect the edition allowlist. Without disabling edition enforcement, a custom layout +or compressor therefore cannot bypass the final wire-ID check. ## How editions change -A frozen edition never changes: neither its component list nor the meaning of its component IDs may be altered. New -components are staged in a **draft** edition, whose contents may change. They become part of the compatibility guarantee -only when that draft is frozen as the next edition in its family. +A frozen edition never changes: neither its membership list nor the meaning of its component IDs may be altered. +Introducing a new serialized object or a reader-visible revision requires a new edition; it is never added +retroactively to an existing edition. A component supplied by an optional plugin creates that edition in the plugin's +independently versioned family. + +Core-maintained objects do not enter `core` directly. When an object is ready for users to try and its wire format is +believed complete, it enters a draft edition in an independently versioned family. Publishing that edition is a +format-stability commitment, not the start of format design: the serialized contract should change only when absolutely +necessary to resolve a problem found during testing. After successful testing, the same object ID and wire contract move +into a new `preview` edition for broad opt-in use, and later into a new `core` edition for use by default. + +A new stable `core` or plugin edition may freeze in the release of its origin project in which it first ships. Until +that release is cut, its version is not known and the declaration keeps `min_library_version: None`. After the release +is cut, the declaration is updated with that newly released version, usually during development of the next release. +This backfills the documented minimum library version; it does not delay the freeze or its read-forever compatibility +guarantee. A component may later be deprecated, meaning that writers stop using it. Readers must continue to support it, so deprecation does not invalidate existing files. +Writer behavior evolves independently from the in-memory representation. A change that an old reader must distinguish +uses a new serialized ID, even when the new deserializer produces the same in-memory array. A serializer may continue +emitting the older ID for values that satisfy its frozen contract; the selected editions validate the ID it emits. + ## How serialized components evolve Editions govern serialized components, not in-memory representations. An in-memory representation may gain capabilities -or be replaced without changing an edition. On read, the plugin registered for a component ID constructs the current -in-memory representation. On write, the implementation selects a component that can represent the value and is allowed -by the selected editions. +or be replaced without changing an edition. Each in-memory array plugin owns the mapping between that representation and +its wire history: -An in-memory representation often has a single serialized component and uses the same ID in memory and on disk, but this -is not required. Multiple component IDs may deserialize into the same in-memory representation. Editions constrain the -ID stored in the file, because that is what the reader must understand. +- the serialized IDs its deserializer recognizes; +- one serializer that returns the appropriate lossless variant as an ID, metadata, buffers, and children; and +- a deserializer that receives the exact ID found in the file and constructs the current in-memory representation. -### Compatible evolution keeps the ID +An in-memory representation often has one serialized ID equal to its in-memory encoding ID, but this is only the simple +case. Editions constrain the ID stored in the file, because that is what an old reader can recognize. -A component may keep its ID only if changes to its wire format are both **backward and forward compatible**: a new -reader must correctly interpret data from an old writer, and an old reader must correctly interpret data from a new -writer. For example, adding an optional field is compatible only if old readers can safely ignore it and new readers use -the correct default when it is absent. +### Reader-visible evolution requires a new ID -Compatible evolution may broaden what the wire format accepts, but it cannot change the meaning of data that existing -readers already accept. Removing or repurposing a field, redefining existing bytes, and requiring information that old -writers did not provide are all incompatible changes. +Any new form that an old reader does not already understand uses a new serialized ID. This includes additive metadata or +children when an old reader would accept the ID but reject or misinterpret the new combination. The ID is the capability +tag: readers do not consult the edition or negotiate a separate version while decoding an array. -### Incompatible evolution requires a new ID +Keeping an ID is safe only when the emitted representation remains within that ID's existing frozen contract. A writer +may choose a different but already-valid encoding of the same contract, and a reader may fix a bug or normalize the old +form into a newer in-memory structure. Neither action expands what the wire ID means. -An incompatible revision is a new component, with a new ID, registry entry, and edition membership. The old component -remains in the registry and must remain readable. The in-memory representation need not change: it can read and write -both components, choosing between them based on the value and the selected editions. +A new wire ID does not normally require a second in-memory array. The current plugin registers every historical ID, +serializes the current value under the oldest allowed lossless one, and deserializes all of them into the current type. +The old ID remains registered forever. If the compressor and serializer cannot preserve one common in-memory +representation and losslessly downgrade it, the change instead needs a new in-memory array, compressor, and +deserializer. Name successive incompatible revisions by appending a version to the same base name: `vortex.foo`, `vortex.foo_v2`, `vortex.foo_v3`. Do not give successor versions descriptive names. A linear naming scheme keeps the component's @@ -135,11 +168,32 @@ metadata includes `lower_part_count`, but readers of this component require that representation gains support for wide decimals, represented by a signed most-significant part and one or more unsigned 64-bit lower parts: -- A single-part array still serializes as `vortex.decimal_byte_parts` with - `lower_part_count = 0`, indistinguishable from files written before the change. -- An array with lower parts uses the new `vortex.decimal_byte_parts_v2` component, initially staged in a draft edition. +- The serializer first tries to construct the old single-signed-child form. If every value can be + represented that way, it emits `vortex.decimal_byte_parts` with `lower_part_count = 0`, even if + the current in-memory array has lower-part children. +- An array that cannot be collapsed into that old form losslessly uses the new + `vortex.decimal_byte_parts_v2` component, initially staged in a draft edition. - A new reader deserializes both IDs into the same in-memory representation. An older reader reports `vortex.decimal_byte_parts_v2` as unknown instead of trying to decode a wire format it does not support. +- When targeting an edition that permits only the old ID, serializing a value that can be collapsed succeeds; an + irreducibly multi-part value fails because no lossless downgrade exists. + +#### Example: Pco 8-bit integers + +The historical `vortex.pco` contract does not include `i8` or `u8`; readers implementing that contract must not be +sent an 8-bit Pco payload under the familiar ID. Adding 8-bit support keeps one current in-memory `Pco` array but adds +`vortex.pco.v2` as a serialized component: + +- The single Pco serializer emits `vortex.pco` for the primitive types covered by the old contract, even when both IDs + are permitted. +- For `i8` or `u8`, the earliest lossless form is `vortex.pco.v2`. A target edition without that ID rejects the write. +- The current deserializer registers both IDs. When given `vortex.pco`, it still rejects an 8-bit dtype; understanding + the v2 payload does not silently broaden the frozen v1 contract. +- The Pco compression scheme can sample and construct 8-bit Pco arrays without consulting editions. Wire selection + remains the serializer's responsibility. + +If writing an older edition must succeed for every input, its compression policy must choose an in-memory encoding +whose serializer has a permitted lossless form. It must not disguise the newer Pco form with the old ID. ### Reading: deserialize into the current representation @@ -148,28 +202,32 @@ in-memory representation rather than preserving a parallel legacy representation interior patches is read as a `Patched` array around a patch-free ALP array. Similarly, old zone maps, including `vortex.stats` layouts, are read by the machinery used for modern `vortex.zoned` layouts. -Readers do not negotiate versions. They resolve the component ID and deserialize it, or report an +Readers do not negotiate versions. They resolve the component ID, pass that exact ID to its deserializer, and either +construct the current in-memory array or report an [unknown-component error](#resolving-an-unknown-component-error). -### Writing: select a permitted component +A current deserializer must preserve each historical ID's contract. Recognizing a newer ID does not authorize it to +accept the newer metadata, child shape, dtype coverage, or buffer interpretation when the file carries an older ID. -Writers choose a component that both represents the current value and belongs to the selected editions. This need not be -the newest component: if an older component can represent the value exactly, the writer may continue to use it. If the -preferred component is not permitted, the writer has two options: +A file contains its array ID, dtype, metadata, children, and buffers. A newer plugin may be registered under both +`vortex.foo` and `vortex.foo_v2`, but an older build is registered only under `vortex.foo`. This is what guarantees that +the older build rejects a v2 file before interpreting its contents. -1. **Translate.** If the value has a lossless translation to a permitted component, use that component. For example, a - newer layout may write its zone statistics using an older statistics schema. -2. **Convert to canonical and recompress.** Otherwise, decompress the data to a canonical representation and recompress - it with the configured compressors, restricted to the selected editions. This is how arrays are handled today: the - writer normalizes each chunk to a canonical representation, then lets the edition-filtered compressor choose the - final encoding. +### Writing: validate the selected component and writer behavior -Both paths use the normal write pipeline and its configured compressors. If neither can express the data using the -selected editions, the write fails. +For each in-memory array, the writer calls its plugin's single serializer. The serializer owns the versioning logic and +returns the appropriate lossless variant. It may change metadata, buffers, and children without constructing a legacy +in-memory array. Returning `None` means the array cannot be serialized. The serialization context then interns the +returned ID, failing the write if that ID is not permitted by the selected editions. + +This selection happens recursively after compression. Compressor output therefore remains an in-memory concern: a +compressor does not label its array with an edition or choose a wire version. Layouts, extension dtypes, and aggregates +perform their analogous compatibility checks at their own serialization boundaries. ### What this means for each kind -- **Arrays.** The array serialization context permits only encodings from the selected editions. +- **Arrays.** The array serialization context permits only wire IDs from the selected editions. The in-memory array's + serializer chooses its lossless representation, and the context rejects it if its ID is not permitted. - **Layouts.** The layout strategy builds the layout tree at write time. When targeting an older edition, it must use structures available in that edition, such as plain chunked data in place of newer auxiliary layouts. - **Extension dtypes.** Before writing any bytes, the file writer recursively validates every extension dtype in the @@ -178,6 +236,66 @@ selected editions, the write fails. function outside the selected editions fails the write. With `allow_unknown`, readers disable a zone map whose aggregate function they do not recognize; ignoring a zone map only reduces pruning and does not affect correctness. +## The `preview` family + +The additive `preview` family is the shared opt-in set for core-maintained components whose serialized contracts have +survived independent testing but are not yet available to the default core writer. Preview currently contains no +components. Adding the first component will create a later preview edition; unrelated work remains in independent +families until it meets the preview compatibility bar. + +## Independently versioned component families + +Components that are ready for focused testing but are not yet ready for the shared preview set advance through their own +families. Optional modules use families such as `tensor`, `zstd`, `spatial`, and `json`. Each family can evolve without +coupling its chronology or selection to unrelated components. + +The wire format is expected to be complete when its first draft edition is published and should change only when +necessary to resolve an issue discovered during testing. If a correction changes what readers must understand, give the +corrected representation a new ID and add a later edition to the same family. Once testing establishes that an object is +ready for broad opt-in use, promote that same ID and serialized contract into a new `preview` edition. Later adoption by +the default writer promotes it into a new `core` edition. + +The default writer does not emit a component merely because its reader understands it. Users opt in by enabling the +edition containing that component. + +## Declaring, freezing, and the edition records + +The default declarations live in `vortex-edition/src/declarations/`, while optional-module declarations live in their +owning crates. Each declared edition is exported as a TOML record under `vortex/editions/`, grouped by family. A record +names the origin library or project whose releases `min_library_version` refers to. Draft records omit that field and +carry no read-forever guarantee. Regenerate the records by running: + +```sh +cargo run -p xtask -- generate-editions +``` + +Changing the declarations follows the edition's lifecycle: + +1. **Create a new family and edition for every new object.** Never add an unrelated serialized + object or reader-visible revision to an existing family. A revision advances the family that + owns its earlier ID. +2. **Publish test-ready work as a draft.** When an object is ready to be tried and its format is + believed complete, give it a wire ID and add it to a draft edition in its family. Change that + format only when necessary to resolve an issue found during testing; a reader-visible + correction gets another ID and a later edition in the same family. +3. **Promote the tested contract to preview.** Once it is ready for broad opt-in use, add the same + object ID and wire contract to a new additive `preview` edition. Promotion must not redesign + the format. +4. **Promote the adopted contract to core.** Once it is ready for use by the default writer, add + the same object ID and wire contract to a new `core` edition with `min_library_version: None` + and regenerate its draft record, then ship it in a release. The edition freezes as part of that + release. Its minimum library version cannot be populated yet because the release version is not + known until the release is cut. +5. **Backfill the released version.** After cutting the release, set `min_library_version` to that + newly released Vortex version — the version that first shipped readers for every member — and + regenerate the records, converting the draft record into a frozen record. This update usually + lands during development of the next release, but it documents the freeze that already + happened; it does not freeze the edition later. +6. **Never touch it again.** A frozen record is immutable: CI (`cargo run -p xtask -- check-editions`) rejects any + change that edits, renames, unfreezes, + or deletes a frozen record, and rejects new editions that do not extend their family's + chronology. To change what writers may emit, declare the next edition instead. + ## Edition registry Registry entries list the edition in which each component first appeared. Later editions in the same family inherit all @@ -187,7 +305,7 @@ earlier components. #### `core2025.05.0` -Minimum Vortex release: `0.36.0`. +Minimum library version: `0.36.0`. - `array`: `fastlanes.bitpacked`, `fastlanes.for`, `vortex.alp`, `vortex.alprd`, `vortex.bool`, `vortex.bytebool`, `vortex.chunked`, `vortex.constant`, `vortex.datetimeparts`, `vortex.decimal`, @@ -199,19 +317,19 @@ Minimum Vortex release: `0.36.0`. #### `core2025.06.0` -Minimum Vortex release: `0.40.0`. +Minimum library version: `0.40.0`. - `array`: `vortex.pco`, `vortex.sequence`, `vortex.zstd` #### `core2025.10.0` -Minimum Vortex release: `0.54.0`. +Minimum library version: `0.54.0`. - `array`: `fastlanes.rle`, `vortex.fixed_size_list`, `vortex.listview`, `vortex.masked` #### `core2026.08.0` -Minimum Vortex release: `0.84.0`. +Minimum library version: `0.84.0`. - `layout`: `vortex.zoned` - `aggregate`: `vortex.bounded_max`, `vortex.bounded_min`, `vortex.max`, `vortex.min`, @@ -219,40 +337,42 @@ Minimum Vortex release: `0.84.0`. #### `core2026.08.1` -Minimum Vortex release: `0.84.0`. - -- `array`: `vortex.map` - -### Draft editions +Minimum library version: `0.84.0`. -Draft component lists may change and have no minimum reader or permanent compatibility guarantee. +- `array`: `vortex.onpair` #### `core2026.08.2` -- `array`: `vortex.parquet.variant`, `vortex.variant` -- `dtype`: `vortex.uuid` +Minimum library version: `0.85.0`. + +- `array`: `vortex.map` #### `core2026.08.3` -- `array`: `vortex.onpair` +Minimum library version: `0.85.0`. + +- `array`: `vortex.parquet.variant`, `vortex.variant` +- `dtype`: `vortex.uuid` -#### `preview2025.05.0` +### Editions without a frozen guarantee -- `array`: `fastlanes.delta` +These editions have no minimum library version. Evolving features advance through new draft editions in their own +families. Their formats are expected to remain compatible unless a defect is serious enough to block promotion into +core. Optional plugin families state their own policy. -#### `preview2026.02.0` +#### `preview2026.08.0` -- `array`: `vortex.zstd_buffers` +This edition currently adds no components. -#### `preview2026.04.0` +#### `tensor2026.04.0` -- `array`: `vortex.patched`, `vortex.tensor.cosine_similarity`, `vortex.tensor.inner_product`, - `vortex.tensor.l2_norm`, `vortex.tensor.normalized` +- `array`: `vortex.tensor.cosine_similarity`, `vortex.tensor.inner_product`, `vortex.tensor.l2_norm`, + `vortex.tensor.normalized` - `dtype`: `vortex.tensor.fixed_shape_tensor`, `vortex.tensor.vector` -#### `preview2026.06.0` +#### `zstd2026.02.0` -- `layout`: `vortex.list` +- `array`: `vortex.zstd_buffers` #### `spatial2026.08.0` diff --git a/encodings/alp/src/alp/plugin.rs b/encodings/alp/src/alp/plugin.rs index c14133109d1..5756dc5a991 100644 --- a/encodings/alp/src/alp/plugin.rs +++ b/encodings/alp/src/alp/plugin.rs @@ -7,17 +7,17 @@ //! This enables zero-cost backward compatibility with previously written datasets. use vortex_array::Array; +use vortex_array::ArrayDeserialization; use vortex_array::ArrayId; use vortex_array::ArrayPlugin; use vortex_array::ArrayRef; +use vortex_array::ArraySerialization; use vortex_array::ArrayVTable; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::Patched; -use vortex_array::buffer::BufferHandle; -use vortex_array::dtype::DType; -use vortex_array::serde::ArrayChildren; use vortex_error::VortexResult; +use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_session::VortexSession; @@ -42,22 +42,29 @@ impl ArrayPlugin for ALPPatchedPlugin { &self, array: &ArrayRef, session: &VortexSession, - ) -> VortexResult>> { + ) -> VortexResult> { // Delegate to ALP's metadata serde - ALP.serialize(array, session) + ArrayPlugin::serialize(&ALP, array, session) } fn deserialize( &self, - dtype: &DType, - len: usize, - metadata: &[u8], - buffers: &[BufferHandle], - children: &dyn ArrayChildren, + parts: ArrayDeserialization<'_>, session: &VortexSession, ) -> VortexResult { + vortex_ensure!( + parts.serialized_id == self.id(), + "ALP plugin does not recognize serialized ID {}", + parts.serialized_id, + ); let alp_array = Array::::try_from_parts(ArrayVTable::deserialize( - &ALP, dtype, len, metadata, buffers, children, session, + &ALP, + parts.dtype, + parts.len, + parts.metadata, + parts.buffers, + parts.children, + session, )?) .map_err(|_| vortex_err!("ALP plugin should only deserialize vortex.alp"))?; @@ -91,6 +98,7 @@ mod tests { use std::f64::consts::PI; use std::sync::LazyLock; + use vortex_array::ArrayDeserialization; use vortex_array::ArrayPlugin; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; @@ -133,7 +141,7 @@ mod tests { let array = alp_encoded.as_array(); - let metadata = SESSION.array_serialize(array)?.unwrap(); + let serialization = SESSION.array_serialize(array)?.unwrap(); let children = array.children(); let buffers = array .buffers() @@ -142,11 +150,14 @@ mod tests { .collect::>(); let deserialized = ALPPatchedPlugin.deserialize( - array.dtype(), - array.len(), - &metadata, - &buffers, - &children, + ArrayDeserialization::new( + ALPPatchedPlugin.id(), + array.dtype(), + array.len(), + &serialization.metadata, + &buffers, + &children, + ), &SESSION, )?; @@ -182,7 +193,7 @@ mod tests { let array = alp_encoded.as_array(); - let metadata = SESSION.array_serialize(array)?.unwrap(); + let serialization = SESSION.array_serialize(array)?.unwrap(); let children = array.children(); let buffers = array .buffers() @@ -191,11 +202,14 @@ mod tests { .collect::>(); let deserialized = ALPPatchedPlugin.deserialize( - array.dtype(), - array.len(), - &metadata, - &buffers, - &children, + ArrayDeserialization::new( + ALPPatchedPlugin.id(), + array.dtype(), + array.len(), + &serialization.metadata, + &buffers, + &children, + ), &SESSION, )?; @@ -213,7 +227,7 @@ mod tests { fn primitive_array_returns_error() { let array = PrimitiveArray::from_iter([1.0f64, 2.0, 3.0]).into_array(); - let metadata = SESSION.array_serialize(&array).unwrap().unwrap(); + let serialization = SESSION.array_serialize(&array).unwrap().unwrap(); let children = array.children(); let buffers = array .buffers() @@ -223,11 +237,14 @@ mod tests { // This panics because PrimitiveArray has no children and ALP requires encoded child. let _result = ALPPatchedPlugin.deserialize( - array.dtype(), - array.len(), - &metadata, - &buffers, - &children, + ArrayDeserialization::new( + ALPPatchedPlugin.id(), + array.dtype(), + array.len(), + &serialization.metadata, + &buffers, + &children, + ), &SESSION, ); } diff --git a/encodings/fastlanes/src/bitpacking/plugin.rs b/encodings/fastlanes/src/bitpacking/plugin.rs index a621d085514..3ff07db7e5c 100644 --- a/encodings/fastlanes/src/bitpacking/plugin.rs +++ b/encodings/fastlanes/src/bitpacking/plugin.rs @@ -7,17 +7,17 @@ //! This enables zero-cost backward compatibility with previously written datasets. use vortex_array::Array; +use vortex_array::ArrayDeserialization; use vortex_array::ArrayId; use vortex_array::ArrayPlugin; use vortex_array::ArrayRef; +use vortex_array::ArraySerialization; use vortex_array::ArrayVTable; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::Patched; -use vortex_array::buffer::BufferHandle; -use vortex_array::dtype::DType; -use vortex_array::serde::ArrayChildren; use vortex_error::VortexResult; +use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_session::VortexSession; @@ -41,22 +41,29 @@ impl ArrayPlugin for BitPackedPatchedPlugin { &self, array: &ArrayRef, session: &VortexSession, - ) -> VortexResult>> { + ) -> VortexResult> { // delegate to BitPacked VTable for serialization - BitPacked.serialize(array, session) + ArrayPlugin::serialize(&BitPacked, array, session) } fn deserialize( &self, - dtype: &DType, - len: usize, - metadata: &[u8], - buffers: &[BufferHandle], - children: &dyn ArrayChildren, + parts: ArrayDeserialization<'_>, session: &VortexSession, ) -> VortexResult { + vortex_ensure!( + parts.serialized_id == self.id(), + "BitPacked plugin does not recognize serialized ID {}", + parts.serialized_id, + ); let bitpacked = Array::::try_from_parts(ArrayVTable::deserialize( - &BitPacked, dtype, len, metadata, buffers, children, session, + &BitPacked, + parts.dtype, + parts.len, + parts.metadata, + parts.buffers, + parts.children, + session, )?) .map_err(|_| vortex_err!("BitPacked plugin should only deserialize fastlanes.bitpacked"))?; @@ -93,6 +100,7 @@ impl ArrayPlugin for BitPackedPatchedPlugin { mod tests { use std::sync::LazyLock; + use vortex_array::ArrayDeserialization; use vortex_array::ArrayPlugin; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; @@ -134,7 +142,7 @@ mod tests { let array = bitpacked.as_array(); - let metadata = SESSION.array_serialize(array)?.unwrap(); + let serialization = SESSION.array_serialize(array)?.unwrap(); let children = array.children(); let buffers = array .buffers() @@ -143,11 +151,14 @@ mod tests { .collect::>(); let deserialized = BitPackedPatchedPlugin.deserialize( - array.dtype(), - array.len(), - &metadata, - &buffers, - &children, + ArrayDeserialization::new( + BitPackedPatchedPlugin.id(), + array.dtype(), + array.len(), + &serialization.metadata, + &buffers, + &children, + ), &SESSION, )?; @@ -184,7 +195,7 @@ mod tests { let array = bitpacked.as_array(); - let metadata = SESSION.array_serialize(array)?.unwrap(); + let serialization = SESSION.array_serialize(array)?.unwrap(); let children = array.children(); let buffers = array .buffers() @@ -193,11 +204,14 @@ mod tests { .collect::>(); let deserialized = BitPackedPatchedPlugin.deserialize( - array.dtype(), - array.len(), - &metadata, - &buffers, - &children, + ArrayDeserialization::new( + BitPackedPatchedPlugin.id(), + array.dtype(), + array.len(), + &serialization.metadata, + &buffers, + &children, + ), &SESSION, )?; @@ -214,7 +228,7 @@ mod tests { fn primitive_array_returns_error() -> VortexResult<()> { let array = PrimitiveArray::from_iter([1i32, 2, 3]).into_array(); - let metadata = SESSION.array_serialize(&array)?.unwrap(); + let serialization = SESSION.array_serialize(&array)?.unwrap(); let children = array.children(); let buffers = array .buffers() @@ -223,11 +237,14 @@ mod tests { .collect::>(); let result = BitPackedPatchedPlugin.deserialize( - array.dtype(), - array.len(), - &metadata, - &buffers, - &children, + ArrayDeserialization::new( + BitPackedPatchedPlugin.id(), + array.dtype(), + array.len(), + &serialization.metadata, + &buffers, + &children, + ), &SESSION, ); diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index ef25c559b99..86b8fd0c34d 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -1033,6 +1033,7 @@ mod test { use fsst::Compressor; use fsst::Symbol; use prost::Message; + use vortex_array::ArrayDeserialization; use vortex_array::ArrayPlugin; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; @@ -1158,19 +1159,22 @@ mod test { let deserialized = ArrayPlugin::deserialize( &FSST, - &DType::Utf8(Nullability::NonNullable), - 2, - &FSSTMetadata { - uncompressed_lengths_ptype: fsst_array - .uncompressed_lengths() - .dtype() - .as_ptype() - .into(), - codes_offsets_ptype: fsst_array.codes_offsets().dtype().as_ptype().into(), - } - .encode_to_vec(), - &buffers, - &children.as_slice(), + ArrayDeserialization::new( + vortex_array::ArrayVTable::id(&FSST), + &DType::Utf8(Nullability::NonNullable), + 2, + &FSSTMetadata { + uncompressed_lengths_ptype: fsst_array + .uncompressed_lengths() + .dtype() + .as_ptype() + .into(), + codes_offsets_ptype: fsst_array.codes_offsets().dtype().as_ptype().into(), + } + .encode_to_vec(), + &buffers, + &children.as_slice(), + ), &array_session(), )?; @@ -1304,20 +1308,23 @@ mod test { let fsst = ArrayPlugin::deserialize( &FSST, - &DType::Utf8(Nullability::NonNullable), - 2, - &FSSTMetadata { - uncompressed_lengths_ptype: fsst_array - .uncompressed_lengths() - .dtype() - .as_ptype() - .into(), - // Legacy array did not store this field, use Protobuf default of 0. - codes_offsets_ptype: 0, - } - .encode_to_vec(), - &buffers, - &children.as_slice(), + ArrayDeserialization::new( + vortex_array::ArrayVTable::id(&FSST), + &DType::Utf8(Nullability::NonNullable), + 2, + &FSSTMetadata { + uncompressed_lengths_ptype: fsst_array + .uncompressed_lengths() + .dtype() + .as_ptype() + .into(), + // Legacy array did not store this field, use Protobuf default of 0. + codes_offsets_ptype: 0, + } + .encode_to_vec(), + &buffers, + &children.as_slice(), + ), &array_session(), )?; diff --git a/encodings/parquet-variant/src/vtable.rs b/encodings/parquet-variant/src/vtable.rs index 302625e70ec..297198df438 100644 --- a/encodings/parquet-variant/src/vtable.rs +++ b/encodings/parquet-variant/src/vtable.rs @@ -410,7 +410,7 @@ mod tests { editions .declare_edition(Edition { id: TEST_EDITION, - min_vortex_version: None, + min_library_version: None, }) .map_err(|error| vortex_err!("{error}"))?; let component_ids = [ diff --git a/encodings/zstd/Cargo.toml b/encodings/zstd/Cargo.toml index 49f15619807..7beacade687 100644 --- a/encodings/zstd/Cargo.toml +++ b/encodings/zstd/Cargo.toml @@ -29,6 +29,7 @@ num-traits = { workspace = true } prost = { workspace = true } vortex-array = { workspace = true } vortex-buffer = { workspace = true } +vortex-edition = { workspace = true } vortex-error = { workspace = true } vortex-mask = { workspace = true } vortex-session = { workspace = true } diff --git a/encodings/zstd/src/editions.rs b/encodings/zstd/src/editions.rs new file mode 100644 index 00000000000..92f096f8a06 --- /dev/null +++ b/encodings/zstd/src/editions.rs @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The `zstd` edition family. +//! +//! Zstd buffer wrapping is opt-in. This module declares its persisted array encoding. + +use vortex_edition::Edition; +use vortex_edition::EditionDeclaration; +use vortex_edition::EditionFamily; +use vortex_edition::EditionId; +use vortex_edition::EditionMember; + +/// The `zstd` family: optional Zstd-backed serialized array representations. +pub static FAMILY: EditionFamily = EditionFamily { + name: "zstd", + origin: "vortex-zstd", + doc: "Optional Zstd-backed serialized array representations. A reader built without \ +`vortex-zstd` cannot resolve these members, so they are versioned independently of `core` and \ +enabled only when the crate is initialized with the corresponding feature.", +}; + +/// The February 2026 draft edition of the `zstd` family. +pub const ZSTD_2026_02: EditionId = EditionId::new("zstd", 2026, 2, 0); + +/// The declaration of [`ZSTD_2026_02`] and the components that join the family at it. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: ZSTD_2026_02, + min_library_version: None, + }, + added: &[EditionMember::array(&"vortex.zstd_buffers")], +}; + +#[cfg(test)] +#[cfg(feature = "unstable_encodings")] +mod tests { + use vortex_edition::EditionError; + use vortex_edition::EditionSessionExt; + use vortex_edition::test_harness::validate_edition; + + use super::*; + + #[test] + fn zstd_edition_is_valid() -> Result<(), EditionError> { + let session = vortex_array::array_session(); + crate::initialize(&session); + validate_edition(&session.editions(), &ZSTD_2026_02) + } +} diff --git a/encodings/zstd/src/lib.rs b/encodings/zstd/src/lib.rs index bb32c2672a0..e2067ffa36a 100644 --- a/encodings/zstd/src/lib.rs +++ b/encodings/zstd/src/lib.rs @@ -23,14 +23,21 @@ pub use array::*; use vortex_array::dtype::proto::dtype as pb; +use vortex_array::session::ArraySessionExt; +#[cfg(feature = "unstable_encodings")] +use vortex_edition::EditionSessionExt; +#[cfg(feature = "unstable_encodings")] +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_err; +use vortex_session::VortexSession; #[cfg(feature = "unstable_encodings")] pub use zstd_buffers::*; mod array; mod compute; +pub mod editions; mod rules; mod slice; #[cfg(feature = "unstable_encodings")] @@ -39,6 +46,30 @@ mod zstd_buffers; #[cfg(test)] mod test; +/// Register the Zstd encodings and their optional edition with a Vortex session. +pub fn initialize(session: &VortexSession) { + session.arrays().register(Zstd); + #[cfg(feature = "unstable_encodings")] + { + session.arrays().register(ZstdBuffers); + if session.editions().find(&editions::ZSTD_2026_02).is_none() { + session + .editions() + .declare_family(&editions::FAMILY) + .map_err(|error| vortex_err!("{error}")) + .vortex_expect("Zstd edition family is valid"); + session + .register_edition(&editions::DECLARATION) + .map_err(|error| vortex_err!("{error}")) + .vortex_expect("Zstd edition declaration is valid"); + } + session + .enable_edition(editions::ZSTD_2026_02) + .map_err(|error| vortex_err!("{error}")) + .vortex_expect("Zstd edition is registered"); + } +} + /// Ensure Vortex metadata agrees with the content size declared by a zstd frame. pub(crate) fn validate_frame_content_size( frame: &[u8], diff --git a/encodings/zstd/src/zstd_buffers.rs b/encodings/zstd/src/zstd_buffers.rs index 5f7f785f71f..a5b273414c5 100644 --- a/encodings/zstd/src/zstd_buffers.rs +++ b/encodings/zstd/src/zstd_buffers.rs @@ -10,6 +10,7 @@ use std::sync::Arc; use prost::Message as _; use vortex_array::Array; +use vortex_array::ArrayDeserialization; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; @@ -60,43 +61,38 @@ impl ZstdBuffers { /// Compress every top-level buffer of `array` independently with zstd. /// - /// Children are preserved as slots and the wrapped array's serialized metadata is stored so the - /// original array can be rebuilt after decompression. + /// The wrapped array's serialized representation is captured so it can be rebuilt after + /// decompression, including any buffers or children selected by its serializer. pub fn compress( array: &ArrayRef, level: i32, session: &VortexSession, ) -> VortexResult { - let encoding_id = array.encoding_id(); - let metadata = session + let serialization = session .array_serialize(array)? .ok_or_else(|| vortex_err!("[ZstdBuffers]: Array does not support serialization"))?; - let buffer_handles = array.buffer_handles(); - let children = array.children(); - let mut compressed_buffers = Vec::with_capacity(buffer_handles.len()); - let mut uncompressed_sizes = Vec::with_capacity(buffer_handles.len()); - let mut buffer_alignments = Vec::with_capacity(buffer_handles.len()); + let mut compressed_buffers = Vec::with_capacity(serialization.buffers.len()); + let mut uncompressed_sizes = Vec::with_capacity(serialization.buffers.len()); + let mut buffer_alignments = Vec::with_capacity(serialization.buffers.len()); let mut compressor = zstd::bulk::Compressor::new(level)?; - // Compression is currently CPU-only, so we gather all buffers on the host. - for handle in &buffer_handles { - buffer_alignments.push(u32::from(handle.alignment())); - let host_buf = handle.clone().try_to_host_sync()?; - uncompressed_sizes.push(host_buf.len() as u64); - let mut compressed = compressor.compress(&host_buf)?; + for buffer in &serialization.buffers { + buffer_alignments.push(u32::from(buffer.alignment())); + uncompressed_sizes.push(buffer.len() as u64); + let mut compressed = compressor.compress(buffer)?; compressed.shrink_to_fit(); compressed_buffers.push(BufferHandle::new_host(ByteBuffer::from(compressed))); } let data = ZstdBuffersData { - inner_encoding_id: encoding_id, - inner_metadata: metadata, + inner_encoding_id: serialization.serialized_id, + inner_metadata: serialization.metadata, compressed_buffers, uncompressed_sizes, buffer_alignments, }; - let slots: ArraySlots = children.into_iter().map(Some).collect(); + let slots: ArraySlots = serialization.children.into_iter().map(Some).collect(); let compressed = Array::try_from_parts( ArrayParts::new(ZstdBuffers, array.dtype().clone(), array.len(), data) .with_slots(slots), @@ -121,11 +117,14 @@ impl ZstdBuffers { let children: Vec = array.slots().iter().flatten().cloned().collect(); inner_vtable.deserialize( - array.dtype(), - array.len(), - &array.data().inner_metadata, - buffer_handles, - &children.as_slice(), + ArrayDeserialization::new( + array.data().inner_encoding_id, + array.dtype(), + array.len(), + &array.data().inner_metadata, + buffer_handles, + &children.as_slice(), + ), session, ) } diff --git a/policy.yml b/policy.yml index 35ad74194d5..8f1449fdbba 100644 --- a/policy.yml +++ b/policy.yml @@ -8,6 +8,7 @@ policy: - a vortex committer has approved - an untouched renovate pull request has an allowed approval - claude or codex authored pull requests have two committer approvals + - core edition changes have an edition owner approval disapproval: options: methods: @@ -86,3 +87,15 @@ approval_rules: count: 2 teams: - "vortex-data/committers" + + - name: core edition changes have an edition owner approval + description: "Changes to core edition declarations require an edition owner." + if: + changed_files: + paths: + - "^vortex-edition/src/declarations/core/.*$" + requires: + count: 1 + users: + - "robert3005" + - "joseph-isaacs" diff --git a/vortex-array/src/array/plugin.rs b/vortex-array/src/array/plugin.rs index 66845eb9a0a..5b4fca39854 100644 --- a/vortex-array/src/array/plugin.rs +++ b/vortex-array/src/array/plugin.rs @@ -6,7 +6,9 @@ use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; +use vortex_buffer::ByteBuffer; use vortex_error::VortexResult; +use vortex_error::vortex_ensure; use vortex_session::VortexSession; use crate::ArrayRef; @@ -21,38 +23,127 @@ use crate::serde::ArrayChildren; /// Reference-counted array plugin. pub type ArrayPluginRef = Arc; -/// Registry trait for ID-based deserialization of arrays. +/// The wire representation produced by an in-memory array's serializer. /// -/// Plugins are registered in the session by their [`ArrayId`]. When a serialized array is -/// encountered, the session resolves the ID to the plugin and calls [`deserialize`] to reconstruct -/// the value as an [`ArrayRef`]. +/// A serializer may reuse the in-memory array's buffers and children with [`Self::from_array`], +/// or return different parts when an older wire representation requires a lossless structural +/// downgrade. +#[derive(Clone, Debug)] +pub struct ArraySerialization { + /// The concrete array ID to write on the wire. + pub serialized_id: ArrayId, + /// Encoding-specific metadata written into the array node. + pub metadata: Vec, + /// Top-level buffers written for this array node. + pub buffers: Vec, + /// Child arrays to serialize recursively. + pub children: Vec, +} + +impl ArraySerialization { + /// Create a wire representation from an ID, metadata, buffers, and children. + pub fn new( + serialized_id: ArrayId, + metadata: Vec, + buffers: Vec, + children: Vec, + ) -> Self { + Self { + serialized_id, + metadata, + buffers, + children, + } + } + + /// Reuse an in-memory array's buffers and children with the supplied serialized metadata. + pub fn from_array(serialized_id: ArrayId, array: &ArrayRef, metadata: Vec) -> Self { + Self::new(serialized_id, metadata, array.buffers(), array.children()) + } +} + +/// The borrowed wire components passed to an array deserializer. +pub struct ArrayDeserialization<'a> { + /// The exact array ID found on the wire. + pub serialized_id: ArrayId, + /// The logical dtype supplied by the containing format. + pub dtype: &'a DType, + /// The logical array length supplied by the containing format. + pub len: usize, + /// Encoding-specific metadata from the array node. + pub metadata: &'a [u8], + /// Top-level buffers referenced by the array node. + pub buffers: &'a [BufferHandle], + /// Lazily decoded child arrays referenced by the array node. + pub children: &'a dyn ArrayChildren, +} + +impl<'a> ArrayDeserialization<'a> { + /// Create borrowed deserialization input from a wire ID and its serialized components. + pub fn new( + serialized_id: ArrayId, + dtype: &'a DType, + len: usize, + metadata: &'a [u8], + buffers: &'a [BufferHandle], + children: &'a dyn ArrayChildren, + ) -> Self { + Self { + serialized_id, + dtype, + len, + metadata, + buffers, + children, + } + } +} + +/// Registry trait for serializing and deserializing an in-memory array representation. /// -/// [`deserialize`]: ArrayPlugin::deserialize +/// A plugin has one [`id`](Self::id) for the in-memory representation and one or more +/// [`serialized_ids`](Self::serialized_ids) for wire representations. Its serializer chooses the +/// wire representation, and the serialization context validates that the chosen ID is permitted +/// before it is written. +/// +/// Every serialized ID is also registered for deserialization. A current plugin may therefore +/// deserialize several historical IDs into the same in-memory representation. A reader that +/// predates a newer ID has no registration for it and reports it as unknown instead of silently +/// interpreting an unsupported representation. pub trait ArrayPlugin: 'static + Send + Sync { - /// Returns the ID for this array encoding. - /// - /// During serde, this is the key the registry uses to find - /// this plugin instance and call the appropriate method on it. + /// Returns the ID of the in-memory array representation handled by this plugin. fn id(&self) -> ArrayId; - /// Serialize the array metadata. + /// Returns the serialized array IDs understood by this plugin, ordered oldest to newest. /// - /// This function will only be called for arrays where the encoding ID matches that of this - /// plugin. - fn serialize(&self, array: &ArrayRef, session: &VortexSession) - -> VortexResult>>; + /// The default uses the in-memory ID as the sole wire ID. Override this for an in-memory array + /// that has multiple serialized variants. IDs retained only for reading may also be included; + /// the single serializer need not select them. + fn serialized_ids(&self) -> Vec { + vec![self.id()] + } - /// Deserialize an array from serialized components. + /// Serialize `array` to its wire representation. /// - /// The returned array doesn't necessary have to match this plugin's encoding ID. This is - /// useful for implementing back-compat logic and deserializing arrays into the new version. + /// This function is called only for arrays whose in-memory encoding matches [`id`](Self::id). + /// The returned ID must be declared by [`serialized_ids`](Self::serialized_ids). Return + /// `Ok(None)` when the array cannot be serialized. + fn serialize( + &self, + array: &ArrayRef, + session: &VortexSession, + ) -> VortexResult>; + + /// Deserialize one recognized wire representation into the current in-memory array. + /// + /// `serialized_id` identifies the exact representation encountered on disk. The returned + /// array does not necessarily have to use this plugin's in-memory ID; this supports legacy + /// representations that are normalized into another current in-memory array. Implementations + /// must validate the contract of that exact ID rather than accepting every form understood by + /// the current in-memory representation under an older ID. fn deserialize( &self, - dtype: &DType, - len: usize, - metadata: &[u8], - buffers: &[BufferHandle], - children: &dyn ArrayChildren, + parts: ArrayDeserialization<'_>, session: &VortexSession, ) -> VortexResult; @@ -80,26 +171,36 @@ impl ArrayPlugin for V { &self, array: &ArrayRef, session: &VortexSession, - ) -> VortexResult>> { - assert_eq!( + ) -> VortexResult> { + vortex_ensure!( + self.id() == array.encoding_id(), + "array plugin {} cannot serialize in-memory array {}", self.id(), array.encoding_id(), - "Invoked for incorrect array ID" ); - V::serialize(array.as_::(), session) + Ok(V::serialize(array.as_::(), session)? + .map(|metadata| ArraySerialization::from_array(self.id(), array, metadata))) } fn deserialize( &self, - dtype: &DType, - len: usize, - metadata: &[u8], - buffers: &[BufferHandle], - children: &dyn ArrayChildren, + parts: ArrayDeserialization<'_>, session: &VortexSession, ) -> VortexResult { + vortex_ensure!( + self.id() == parts.serialized_id, + "array plugin {} does not recognize serialized ID {}", + self.id(), + parts.serialized_id, + ); Ok(Array::::try_from_parts(V::deserialize( - self, dtype, len, metadata, buffers, children, session, + self, + parts.dtype, + parts.len, + parts.metadata, + parts.buffers, + parts.children, + session, )?)? .into_array()) } diff --git a/vortex-array/src/arrays/scalar_fn/plugin.rs b/vortex-array/src/arrays/scalar_fn/plugin.rs index 044f79362cd..46c1a597f9a 100644 --- a/vortex-array/src/arrays/scalar_fn/plugin.rs +++ b/vortex-array/src/arrays/scalar_fn/plugin.rs @@ -2,16 +2,18 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_error::VortexResult; +use vortex_error::vortex_ensure; use vortex_session::VortexSession; +use crate::ArrayDeserialization; use crate::ArrayId; use crate::ArrayPlugin; use crate::ArrayRef; +use crate::ArraySerialization; use crate::IntoArray; use crate::arrays::ScalarFnArray; use crate::arrays::scalar_fn::ExactScalarFn; use crate::arrays::scalar_fn::ScalarFnArrayView; -use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::scalar_fn::ScalarFnVTable; use crate::scalar_fn::TypedScalarFnInstance; @@ -64,27 +66,38 @@ impl ArrayPlugin for ScalarFnArrayPlugi &self, array: &ArrayRef, session: &VortexSession, - ) -> VortexResult>> { + ) -> VortexResult> { // We serialize the scalar function options, along with any scalar function array data. let scalar_fn = array.as_::>(); - ::serialize(&self.0, &scalar_fn, session) + Ok( + ::serialize(&self.0, &scalar_fn, session)? + .map(|metadata| ArraySerialization::from_array(self.id(), array, metadata)), + ) } fn deserialize( &self, - dtype: &DType, - len: usize, - metadata: &[u8], - _buffers: &[BufferHandle], - children: &dyn ArrayChildren, + parts: ArrayDeserialization<'_>, session: &VortexSession, ) -> VortexResult { - let parts = ::deserialize( - &self.0, dtype, len, metadata, children, session, + vortex_ensure!( + parts.serialized_id == self.id(), + "scalar function array plugin {} does not recognize serialized ID {}", + self.id(), + parts.serialized_id, + ); + let len = parts.len; + let scalar_parts = ::deserialize( + &self.0, + parts.dtype, + parts.len, + parts.metadata, + parts.children, + session, )?; Ok(ScalarFnArray::try_new_with_len( - TypedScalarFnInstance::new(self.0.clone(), parts.options).erased(), - parts.children, + TypedScalarFnInstance::new(self.0.clone(), scalar_parts.options).erased(), + scalar_parts.children, len, )? .into_array()) diff --git a/vortex-array/src/serde.rs b/vortex-array/src/serde.rs index f84ec182269..eb763ac924d 100644 --- a/vortex-array/src/serde.rs +++ b/vortex-array/src/serde.rs @@ -30,6 +30,7 @@ use vortex_utils::aliases::hash_map::HashMap; use crate::ArrayContext; use crate::ArrayRef; use crate::ArraySlots; +use crate::array::ArrayDeserialization; use crate::array::ArrayId; use crate::array::new_foreign_array; use crate::buffer::BufferHandle; @@ -65,11 +66,10 @@ impl ArrayRef { session: &VortexSession, options: &SerializeOptions, ) -> VortexResult> { - // Collect all array buffers - let array_buffers = self - .depth_first_traversal() - .flat_map(|f| f.buffers()) - .collect::>(); + // Resolve the wire representation once. Serializers may choose historical IDs and may + // provide downgraded buffers or children that differ from the in-memory array tree. + let root = ArrayNodeFlatBuffer::try_new(ctx, session, self)?; + let array_buffers = root.array.buffers(); // Allocate result buffers, including a possible padding buffer for each. let mut buffers = vec![]; @@ -121,7 +121,6 @@ impl ArrayRef { // Set up the flatbuffer builder let mut fbb = FlatBufferBuilder::new(); - let root = ArrayNodeFlatBuffer::try_new(ctx, session, self)?; let fb_root = root.try_write_flatbuffer(&mut fbb)?; let fb_buffers = fbb.create_vector(&fb_buffers); @@ -158,20 +157,74 @@ impl ArrayRef { } } +#[derive(Clone, Debug)] +struct ArraySerializationTree { + source: ArrayRef, + serialized_id: ArrayId, + metadata: Vec, + buffers: Vec, + children: Vec, +} + +impl ArraySerializationTree { + fn try_new(session: &VortexSession, source: &ArrayRef) -> VortexResult { + let Some(serialization) = session.array_serialize(source)? else { + vortex_bail!( + "Array {} does not support serialization", + source.encoding_id() + ); + }; + let children = serialization + .children + .iter() + .map(|child| Self::try_new(session, child)) + .collect::>>()?; + + Ok(Self { + source: source.clone(), + serialized_id: serialization.serialized_id, + metadata: serialization.metadata, + buffers: serialization.buffers, + children, + }) + } + + fn nbuffers_recursive(&self) -> usize { + self.buffers.len() + + self + .children + .iter() + .map(Self::nbuffers_recursive) + .sum::() + } + + fn buffers(&self) -> Vec { + let mut buffers = Vec::with_capacity(self.nbuffers_recursive()); + self.append_buffers(&mut buffers); + buffers + } + + fn append_buffers(&self, buffers: &mut Vec) { + buffers.extend(self.buffers.iter().cloned()); + for child in &self.children { + child.append_buffers(buffers); + } + } +} + /// A utility struct for creating an [`fba::ArrayNode`] flatbuffer. pub struct ArrayNodeFlatBuffer<'a> { ctx: &'a ArrayContext, - session: &'a VortexSession, - array: &'a ArrayRef, - buffer_idx: u16, + array: ArraySerializationTree, } impl<'a> ArrayNodeFlatBuffer<'a> { pub fn try_new( ctx: &'a ArrayContext, session: &'a VortexSession, - array: &'a ArrayRef, + array: &ArrayRef, ) -> VortexResult { + let array = ArraySerializationTree::try_new(session, array)?; let n_buffers_recursive = array.nbuffers_recursive(); if n_buffers_recursive > u16::MAX as usize { vortex_bail!( @@ -179,51 +232,42 @@ impl<'a> ArrayNodeFlatBuffer<'a> { n_buffers_recursive ); }; - Ok(Self { - ctx, - session, - array, - buffer_idx: 0, - }) + Ok(Self { ctx, array }) } pub fn try_write_flatbuffer<'fb>( &self, fbb: &mut FlatBufferBuilder<'fb>, ) -> VortexResult>> { - let encoding_idx = self.ctx.intern(&self.array.encoding_id()).ok_or_else(|| { - vortex_err!( - "Array encoding {} not permitted by ctx", - self.array.encoding_id() - ) - })?; + self.try_write_node(fbb, &self.array, 0) + } - let metadata_bytes = self.session.array_serialize(self.array)?.ok_or_else(|| { + fn try_write_node<'fb>( + &self, + fbb: &mut FlatBufferBuilder<'fb>, + array: &ArraySerializationTree, + buffer_idx: u16, + ) -> VortexResult>> { + let encoding_idx = self.ctx.intern(&array.serialized_id).ok_or_else(|| { vortex_err!( - "Array {} does not support serialization", - self.array.encoding_id() + "Serialized array ID {} not permitted by ctx", + array.serialized_id ) })?; - let metadata = Some(fbb.create_vector(metadata_bytes.as_slice())); + + let metadata = Some(fbb.create_vector(array.metadata.as_slice())); // Assign buffer indices for all child arrays. - let nbuffers = u16::try_from(self.array.nbuffers()) + let nbuffers = u16::try_from(array.buffers.len()) .map_err(|_| vortex_err!("Array can have at most u16::MAX buffers"))?; - let mut child_buffer_idx = self.buffer_idx + nbuffers; + let mut child_buffer_idx = buffer_idx + nbuffers; - let children = self - .array - .children() + let children = array + .children .iter() .map(|child| { // Update the number of buffers required. - let msg = ArrayNodeFlatBuffer { - ctx: self.ctx, - session: self.session, - array: child, - buffer_idx: child_buffer_idx, - } - .try_write_flatbuffer(fbb)?; + let msg = self.try_write_node(fbb, child, child_buffer_idx)?; child_buffer_idx = u16::try_from(child.nbuffers_recursive()) .ok() @@ -235,8 +279,8 @@ impl<'a> ArrayNodeFlatBuffer<'a> { .collect::>>()?; let children = Some(fbb.create_vector(&children)); - let buffers = Some(fbb.create_vector_from_iter((0..nbuffers).map(|i| i + self.buffer_idx))); - let stats = Some(self.array.statistics().write_flatbuffer(fbb)?); + let buffers = Some(fbb.create_vector_from_iter((0..nbuffers).map(|i| i + buffer_idx))); + let stats = Some(array.source.statistics().write_flatbuffer(fbb)?); Ok(fba::ArrayNode::create( fbb, @@ -336,8 +380,17 @@ impl SerializedArray { let buffers = self.collect_buffers()?; - let decoded = - plugin.deserialize(dtype, len, self.metadata(), &buffers, &children, session)?; + let decoded = plugin.deserialize( + ArrayDeserialization::new( + encoding_id, + dtype, + len, + self.metadata(), + &buffers, + &children, + ), + session, + )?; assert_eq!( decoded.len(), @@ -716,13 +769,236 @@ impl TryFrom for SerializedArray { #[cfg(test)] mod tests { + use std::sync::atomic::AtomicUsize; + use std::sync::atomic::Ordering; + use vortex_buffer::ByteBufferMut; + use vortex_error::vortex_ensure; + use vortex_session::registry::CachedId; use super::*; + use crate::Array; + use crate::ArrayPlugin; + use crate::ArraySerialization; + use crate::ArrayVTable; use crate::IntoArray; use crate::array_session; + use crate::arrays::Primitive; use crate::arrays::PrimitiveArray; + static SERIALIZER_CALLS: AtomicUsize = AtomicUsize::new(0); + + fn old_primitive_id() -> ArrayId { + ArrayVTable::id(&Primitive) + } + + fn new_primitive_id() -> ArrayId { + static ID: CachedId = CachedId::new("vortex.test.primitive_v2"); + *ID + } + + #[derive(Debug)] + struct VersionedPrimitivePlugin; + + impl ArrayPlugin for VersionedPrimitivePlugin { + fn id(&self) -> ArrayId { + old_primitive_id() + } + + fn serialized_ids(&self) -> Vec { + vec![old_primitive_id(), new_primitive_id()] + } + + fn serialize( + &self, + array: &ArrayRef, + _session: &VortexSession, + ) -> VortexResult> { + vortex_ensure!( + array.encoding_id() == self.id(), + "versioned primitive serializer received {}", + array.encoding_id(), + ); + + let serialized_id = if array.len() <= 4 { + old_primitive_id() + } else { + new_primitive_id() + }; + + Ok(Some(ArraySerialization::from_array( + serialized_id, + array, + vec![], + ))) + } + + fn deserialize( + &self, + parts: ArrayDeserialization<'_>, + session: &VortexSession, + ) -> VortexResult { + vortex_ensure!( + parts.serialized_id == old_primitive_id() + || parts.serialized_id == new_primitive_id(), + "versioned primitive deserializer does not recognize {}", + parts.serialized_id, + ); + vortex_ensure!( + parts.serialized_id != old_primitive_id() || parts.len <= 4, + "old primitive wire ID cannot represent length {}", + parts.len, + ); + Ok(Array::::try_from_parts(ArrayVTable::deserialize( + &Primitive, + parts.dtype, + parts.len, + parts.metadata, + parts.buffers, + parts.children, + session, + )?)? + .into_array()) + } + } + + #[derive(Debug)] + struct CountingVersionedPrimitivePlugin; + + impl ArrayPlugin for CountingVersionedPrimitivePlugin { + fn id(&self) -> ArrayId { + VersionedPrimitivePlugin.id() + } + + fn serialized_ids(&self) -> Vec { + VersionedPrimitivePlugin.serialized_ids() + } + + fn serialize( + &self, + array: &ArrayRef, + session: &VortexSession, + ) -> VortexResult> { + SERIALIZER_CALLS.fetch_add(1, Ordering::Relaxed); + VersionedPrimitivePlugin.serialize(array, session) + } + + fn deserialize( + &self, + parts: ArrayDeserialization<'_>, + session: &VortexSession, + ) -> VortexResult { + VersionedPrimitivePlugin.deserialize(parts, session) + } + } + + fn versioned_primitive_session() -> VortexSession { + let session = array_session(); + session.arrays().register(VersionedPrimitivePlugin); + session + } + + fn restricted_context(ids: &[ArrayId]) -> ArrayContext { + ArrayContext::new(ids.to_vec()).with_allowed_ids(ids.iter().copied().collect()) + } + + fn serialize_blob( + array: &ArrayRef, + ctx: &ArrayContext, + session: &VortexSession, + ) -> VortexResult { + let mut blob = ByteBufferMut::empty(); + for buffer in array.serialize(ctx, session, &SerializeOptions::default())? { + blob.extend_from_slice(buffer.as_ref()); + } + Ok(blob.freeze()) + } + + #[test] + fn one_serializer_selects_the_earliest_lossless_wire_id() -> VortexResult<()> { + let session = array_session(); + session.arrays().register(CountingVersionedPrimitivePlugin); + let ctx = restricted_context(&[old_primitive_id(), new_primitive_id()]); + let array = PrimitiveArray::from_iter([1i32, 2, 3, 4]).into_array(); + + SERIALIZER_CALLS.store(0, Ordering::Relaxed); + let serialized = SerializedArray::try_from(serialize_blob(&array, &ctx, &session)?)?; + assert_eq!(SERIALIZER_CALLS.load(Ordering::Relaxed), 1); + assert_eq!( + ReadContext::new(ctx.to_ids()).resolve(serialized.encoding_id()), + Some(old_primitive_id()) + ); + Ok(()) + } + + #[test] + fn serializer_uses_a_newer_id_only_when_the_old_variant_cannot_represent_the_value() + -> VortexResult<()> { + let session = versioned_primitive_session(); + let ctx = restricted_context(&[old_primitive_id(), new_primitive_id()]); + let array = PrimitiveArray::from_iter(0..8i32).into_array(); + let serialized = SerializedArray::try_from(serialize_blob(&array, &ctx, &session)?)?; + let read_ctx = ReadContext::new(ctx.to_ids()); + + assert_eq!( + read_ctx.resolve(serialized.encoding_id()), + Some(new_primitive_id()) + ); + let decoded = serialized.decode(array.dtype(), array.len(), &read_ctx, &session)?; + assert_eq!(decoded.encoding_id(), old_primitive_id()); + Ok(()) + } + + #[test] + fn serialization_fails_when_serialized_id_is_not_permitted() -> VortexResult<()> { + let session = versioned_primitive_session(); + let ctx = restricted_context(&[old_primitive_id()]); + let array = PrimitiveArray::from_iter(0..8i32).into_array(); + + let error = array + .serialize(&ctx, &session, &SerializeOptions::default()) + .expect_err("the serialized ID is not permitted"); + assert!(error.to_string().contains("not permitted by ctx")); + Ok(()) + } + + #[test] + fn old_reader_rejects_a_new_serialized_id() -> VortexResult<()> { + let writer_session = versioned_primitive_session(); + let ctx = restricted_context(&[new_primitive_id()]); + let array = PrimitiveArray::from_iter(0..8i32).into_array(); + let serialized = SerializedArray::try_from(serialize_blob(&array, &ctx, &writer_session)?)?; + let read_ctx = ReadContext::new(ctx.to_ids()); + + let old_session = array_session(); + let error = serialized + .decode(array.dtype(), array.len(), &read_ctx, &old_session) + .expect_err("an old reader must not recognize the new wire ID"); + assert!(error.to_string().contains("Unknown encoding")); + Ok(()) + } + + #[test] + fn deserializer_enforces_the_exact_wire_id_contract() -> VortexResult<()> { + let session = versioned_primitive_session(); + let write_ctx = restricted_context(&[new_primitive_id()]); + let array = PrimitiveArray::from_iter(0..8i32).into_array(); + let serialized = SerializedArray::try_from(serialize_blob(&array, &write_ctx, &session)?)?; + + // Interpret the encoded index as the old ID to simulate a file that uses the old tag for + // a representation outside that tag's frozen contract. + let error = serialized + .decode( + array.dtype(), + array.len(), + &ReadContext::new([old_primitive_id()]), + &session, + ) + .expect_err("the old wire contract must be enforced by the current deserializer"); + assert!(error.to_string().contains("old primitive wire ID")); + Ok(()) + } + /// A corrupt array tree can declare a buffer that extends past the backing segment. Slicing /// such a buffer must return a [`VortexError`] rather than panicking (see issue #8819). #[test] diff --git a/vortex-array/src/session/mod.rs b/vortex-array/src/session/mod.rs index 2f3fbb9e4e7..e9d0d36a55f 100644 --- a/vortex-array/src/session/mod.rs +++ b/vortex-array/src/session/mod.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use vortex_error::VortexResult; use vortex_error::vortex_bail; +use vortex_error::vortex_ensure; use vortex_session::ArcSwapMap; use vortex_session::SessionExt; use vortex_session::SessionGuard; @@ -13,8 +14,10 @@ use vortex_session::SessionVar; use vortex_session::registry::Id; use crate::ArrayRef; +use crate::array::ArrayId; use crate::array::ArrayPlugin; use crate::array::ArrayPluginRef; +use crate::array::ArraySerialization; use crate::arrays::Bool; use crate::arrays::Chunked; use crate::arrays::Constant; @@ -40,14 +43,17 @@ pub type ArrayRegistry = ArcSwapMap; #[derive(Clone, Debug)] pub struct ArraySession { - /// The set of registered array encodings. + /// Deserializers keyed by the array ID found on the wire. registry: ArrayRegistry, + /// Serializers keyed by the in-memory array encoding ID. + serializers: ArrayRegistry, } impl ArraySession { pub fn empty() -> ArraySession { Self { registry: ArrayRegistry::default(), + serializers: ArrayRegistry::default(), } } @@ -55,10 +61,20 @@ impl ArraySession { &self.registry } - /// Register a new array encoding, replacing any existing encoding with the same ID. + /// Register an in-memory array plugin and all of its recognized serialized IDs. + /// + /// This replaces any serializer with the same in-memory ID and any deserializer registered + /// under one of [`ArrayPlugin::serialized_ids`]. pub fn register(&self, plugin: P) { - self.registry - .insert(plugin.id(), Arc::new(plugin) as ArrayPluginRef); + let plugin = Arc::new(plugin) as ArrayPluginRef; + self.serializers.insert(plugin.id(), Arc::clone(&plugin)); + for serialized_id in plugin.serialized_ids() { + self.registry.insert(serialized_id, Arc::clone(&plugin)); + } + } + + fn serializer(&self, id: &ArrayId) -> Option { + self.serializers.get(id) } } @@ -66,6 +82,7 @@ impl Default for ArraySession { fn default() -> Self { let this = ArraySession { registry: ArrayRegistry::default(), + serializers: ArrayRegistry::default(), }; // Register the canonical encodings. @@ -113,15 +130,26 @@ pub trait ArraySessionExt: SessionExt { } /// Serialize an array using a plugin from the registry. - fn array_serialize(&self, array: &ArrayRef) -> VortexResult>> { - let Some(plugin) = self.arrays().registry.get(&array.encoding_id()) else { + fn array_serialize(&self, array: &ArrayRef) -> VortexResult> { + let Some(plugin) = self.arrays().serializer(&array.encoding_id()) else { vortex_bail!( - "Array {} is not registered for serializations", + "Array {} is not registered for serialization", array.encoding_id() ); }; - plugin.serialize(array, &self.session()) + let Some(serialization) = plugin.serialize(array, &self.session())? else { + return Ok(None); + }; + vortex_ensure!( + plugin + .serialized_ids() + .contains(&serialization.serialized_id), + "array serializer {} produced undeclared serialized ID {}", + array.encoding_id(), + serialization.serialized_id, + ); + Ok(Some(serialization)) } } @@ -141,6 +169,7 @@ mod tests { let session = VortexSession::empty().with::(); assert!(session.arrays().registry().contains_key(&Bool.id())); + assert!(session.arrays().serializer(&Bool.id()).is_some()); } #[test] @@ -148,5 +177,6 @@ mod tests { let session = VortexSession::empty().with_some(ArraySession::empty()); assert!(!session.arrays().registry().contains_key(&Bool.id())); + assert!(session.arrays().serializer(&Bool.id()).is_none()); } } diff --git a/vortex-bench/src/conversions.rs b/vortex-bench/src/conversions.rs index f9c4ee63734..de0a163abf1 100644 --- a/vortex-bench/src/conversions.rs +++ b/vortex-bench/src/conversions.rs @@ -66,6 +66,7 @@ use wkb::writer::write_geometry; use crate::CompactionStrategy; use crate::Format; use crate::SESSION; +use crate::benchmark_write_options; use crate::utils::file::idempotent_async; /// Memory budget per concurrent conversion stream in GB. This is somewhat arbitary. @@ -255,7 +256,7 @@ fn write_options_for( for name in binary_fields { builder = builder.with_field_writer(FieldPath::from_name(name), no_dict_layout()); } - SESSION.write_options().with_strategy(builder.build()) + benchmark_write_options(SESSION.write_options()).with_strategy(builder.build()) } /// A chunked + compressed layout that skips dictionary encoding for opaque `Binary` blobs. diff --git a/vortex-bench/src/datasets/taxi_data.rs b/vortex-bench/src/datasets/taxi_data.rs index e592d84aec9..bc1da6ea1aa 100644 --- a/vortex-bench/src/datasets/taxi_data.rs +++ b/vortex-bench/src/datasets/taxi_data.rs @@ -18,6 +18,7 @@ use crate::CompactionStrategy; use crate::Format; use crate::IdempotentPath; use crate::SESSION; +use crate::benchmark_write_options; use crate::conversions::parquet_to_vortex_chunks; use crate::datasets::Dataset; use crate::datasets::data_downloads::download_data; @@ -99,8 +100,7 @@ pub async fn taxi_data_vortex() -> Result { let data = parquet_to_vortex_chunks(taxi_data_parquet().await?).await?; - SESSION - .write_options() + benchmark_write_options(SESSION.write_options()) .write(&mut output_file, data.into_array().to_array_stream()) .await?; output_file.flush().await?; diff --git a/vortex-bench/src/downloadable_dataset.rs b/vortex-bench/src/downloadable_dataset.rs index cd73ba5af84..ba67b741bdb 100644 --- a/vortex-bench/src/downloadable_dataset.rs +++ b/vortex-bench/src/downloadable_dataset.rs @@ -12,6 +12,7 @@ use vortex::file::WriteOptionsSessionExt; use crate::IdempotentPath; use crate::SESSION; +use crate::benchmark_write_options; use crate::conversions::parquet_to_vortex_chunks; use crate::datasets::Dataset; use crate::datasets::data_downloads::download_data; @@ -65,8 +66,7 @@ impl Dataset for DownloadableDataset { let data = parquet_to_vortex_chunks(parquet).await?; idempotent_async(&vortex, async |path| -> anyhow::Result<()> { - SESSION - .write_options() + benchmark_write_options(SESSION.write_options()) .write( &mut File::create(path) .await diff --git a/vortex-bench/src/lib.rs b/vortex-bench/src/lib.rs index d68cdc43e93..7778742e0b9 100644 --- a/vortex-bench/src/lib.rs +++ b/vortex-bench/src/lib.rs @@ -250,6 +250,7 @@ pub enum CompactionStrategy { impl CompactionStrategy { pub fn apply_options(&self, options: VortexWriteOptions) -> VortexWriteOptions { + let options = benchmark_write_options(options); match self { CompactionStrategy::Compact => options.with_strategy( WriteStrategyBuilder::default() @@ -261,6 +262,21 @@ impl CompactionStrategy { } } +/// Apply the write policy shared by Vortex benchmarks. +/// +/// Benchmark builds that enable unstable encodings intentionally exercise all registered array +/// encodings, including those that do not yet belong to an edition. +pub fn benchmark_write_options(options: VortexWriteOptions) -> VortexWriteOptions { + #[cfg(feature = "unstable_encodings")] + { + options.disable_editions() + } + #[cfg(not(feature = "unstable_encodings"))] + { + options + } +} + /// Verify that local data has already been prepared for the requested benchmark formats. /// /// Engine-specific benchmark binaries call this before running queries. Data generation itself diff --git a/vortex-bench/src/public_bi.rs b/vortex-bench/src/public_bi.rs index 3e651df5bb7..cf3b6fac6e4 100644 --- a/vortex-bench/src/public_bi.rs +++ b/vortex-bench/src/public_bi.rs @@ -41,6 +41,7 @@ use crate::Format; use crate::IdempotentPath; use crate::SESSION; use crate::TableSpec; +use crate::benchmark_write_options; use crate::conversions::parquet_to_vortex_chunks; use crate::datasets::Dataset; use crate::datasets::data_downloads::decompress_bz2; @@ -364,8 +365,7 @@ impl PBIData { let data = parquet_to_vortex_chunks(parquet).await?; let vortex_file = idempotent_async(&vortex, async |output_path| -> anyhow::Result<()> { - SESSION - .write_options() + benchmark_write_options(SESSION.write_options()) .write( &mut File::create(output_path) .await diff --git a/vortex-bench/src/spatialbench/datagen/native.rs b/vortex-bench/src/spatialbench/datagen/native.rs index 8a7acc7d2fd..ba5892d30c1 100644 --- a/vortex-bench/src/spatialbench/datagen/native.rs +++ b/vortex-bench/src/spatialbench/datagen/native.rs @@ -38,6 +38,7 @@ use vortex_arrow::ArrowSessionExt; use super::table::GeometryKind; use super::table::Table; use crate::SESSION; +use crate::benchmark_write_options; use crate::utils::file::idempotent_async; fn geoarrow_metadata() -> Arc { @@ -58,8 +59,7 @@ pub async fn write_native_vortex( let dtype = chunks[0].dtype().clone(); let chunked = ChunkedArray::try_new(chunks, dtype)?.into_array(); let mut file = TokioFile::create(&path).await?; - SESSION - .write_options() + benchmark_write_options(SESSION.write_options()) .write(&mut file, chunked.to_array_stream()) .await?; tracing::info!(path = %path.display(), table = table.name(), "wrote native geometry table"); diff --git a/vortex-btrblocks/src/schemes/integer/rle.rs b/vortex-btrblocks/src/schemes/integer/rle.rs index ad87532b782..7cda599f5b5 100644 --- a/vortex-btrblocks/src/schemes/integer/rle.rs +++ b/vortex-btrblocks/src/schemes/integer/rle.rs @@ -25,6 +25,8 @@ use vortex_fastlanes::RLE; use vortex_fastlanes::RLEArrayExt; use vortex_fastlanes::RLEArraySlotsExt; +#[cfg(feature = "unstable_encodings")] +use super::DeltaScheme; use super::RUN_LENGTH_THRESHOLD; use crate::ArrayAndStats; use crate::CascadingCompressor; @@ -60,38 +62,28 @@ pub(crate) fn rle_compress( exec_ctx, )?; - // Delta is an unstable encoding, once we deem it stable we can switch over to this always. - #[cfg(feature = "unstable_encodings")] - let compressed_indices = { - let rle_indices_primitive = rle_array - .indices() - .clone() - .execute::(exec_ctx)? - .narrow(exec_ctx)?; - try_compress_delta( - compressor, - &rle_indices_primitive.into_array(), - &compress_ctx, - scheme.id(), - 1, - exec_ctx, - )? - }; - - #[cfg(not(feature = "unstable_encodings"))] let compressed_indices = { let rle_indices_primitive = rle_array .indices() .clone() .execute::(exec_ctx)? .narrow(exec_ctx)?; - compressor.compress_child( - &rle_indices_primitive.into_array(), - &compress_ctx, - scheme.id(), - 1, - exec_ctx, - )? + let rle_indices = rle_indices_primitive.into_array(); + #[cfg(feature = "unstable_encodings")] + if compressor.has_scheme(DeltaScheme::default().id()) { + try_compress_delta( + compressor, + &rle_indices, + &compress_ctx, + scheme.id(), + 1, + exec_ctx, + )? + } else { + compressor.compress_child(&rle_indices, &compress_ctx, scheme.id(), 1, exec_ctx)? + } + #[cfg(not(feature = "unstable_encodings"))] + compressor.compress_child(&rle_indices, &compress_ctx, scheme.id(), 1, exec_ctx)? }; let rle_offsets_primitive = rle_array diff --git a/vortex-btrblocks/src/schemes/string/onpair.rs b/vortex-btrblocks/src/schemes/string/onpair.rs index 75ec97d2191..a1bc8643775 100644 --- a/vortex-btrblocks/src/schemes/string/onpair.rs +++ b/vortex-btrblocks/src/schemes/string/onpair.rs @@ -26,6 +26,7 @@ use crate::CascadingCompressor; use crate::CompressorContext; use crate::Scheme; use crate::SchemeExt; +use crate::schemes::integer::DeltaScheme; use crate::schemes::integer::try_compress_delta; /// OnPair short-string compression (dict-12). @@ -177,6 +178,9 @@ fn compress_offsets_child( if narrowed.len() < OFFSETS_DELTA_MIN_LEN { return Ok(plain); } + if !compressor.has_scheme(DeltaScheme::default().id()) { + return Ok(plain); + } let delta = try_compress_delta( compressor, &narrowed, diff --git a/vortex-compressor/src/compressor/mod.rs b/vortex-compressor/src/compressor/mod.rs index a661970950c..219b67e2519 100644 --- a/vortex-compressor/src/compressor/mod.rs +++ b/vortex-compressor/src/compressor/mod.rs @@ -65,6 +65,13 @@ impl CascadingCompressor { root_exclusions, } } + + /// Returns whether the compressor was configured with `scheme`. + pub fn has_scheme(&self, scheme: SchemeId) -> bool { + self.schemes + .iter() + .any(|candidate| candidate.id() == scheme) + } } // NB: Cascading compression logic is located in `vortex-compressor/src/compressor/cascade.rs`. diff --git a/vortex-edition/src/declarations/core/mod.rs b/vortex-edition/src/declarations/core/mod.rs new file mode 100644 index 00000000000..93615b93dfc --- /dev/null +++ b/vortex-edition/src/declarations/core/mod.rs @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The `core` edition family: serialized components available to the default file writer. +//! +//! One module per edition, each declaring the edition and the serialized components that join the +//! family at it; members of earlier editions are inherited and never restated. + +use crate::EditionFamily; + +/// The `core` family: serialized components available by default. +pub static FAMILY: EditionFamily = EditionFamily { + name: "core", + origin: "vortex", + doc: "The serialized components available to the default file writer. Each array ID names a \ +wire representation that old readers either recognize or reject; several IDs may deserialize \ +into one current in-memory array. Every core edition freezes, and a frozen edition carries a \ +read-forever guarantee: a file written with it stays readable by every later Vortex release. \ +New core objects first undergo testing in independently versioned families, then join preview for \ +broad opt-in use, and finally join core with the same IDs and wire contracts. An edition may \ +freeze in the release that cuts it; after that release version is known, the declaration is \ +backfilled with it as the minimum. A frozen edition never changes.", +}; + +pub mod v2025_05; +pub mod v2025_06; +pub mod v2025_10; +pub mod v2026_08; +pub mod v2026_08_2; +pub mod v2026_08_3; + +pub use v2025_05::CORE_2025_05_0; +pub use v2025_06::CORE_2025_06_0; +pub use v2025_10::CORE_2025_10_0; +pub use v2026_08::CORE_2026_08_0; +pub use v2026_08::CORE_2026_08_1; +pub use v2026_08_2::CORE_2026_08_2; +pub use v2026_08_3::CORE_2026_08_3; diff --git a/vortex/src/editions/core/v2025_05.rs b/vortex-edition/src/declarations/core/v2025_05.rs similarity index 92% rename from vortex/src/editions/core/v2025_05.rs rename to vortex-edition/src/declarations/core/v2025_05.rs index 235b25e0d3c..38ca6d75a10 100644 --- a/vortex/src/editions/core/v2025_05.rs +++ b/vortex-edition/src/declarations/core/v2025_05.rs @@ -3,10 +3,10 @@ //! The baseline `core` edition: stable serialized components writable by Vortex 0.36.0. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; -use vortex_edition::EditionMember; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; +use crate::EditionMember; /// The first edition of the `core` family, matching the first stable Vortex file release. pub const CORE_2025_05_0: EditionId = EditionId::new("core", 2025, 5, 0); @@ -15,7 +15,7 @@ pub const CORE_2025_05_0: EditionId = EditionId::new("core", 2025, 5, 0); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: CORE_2025_05_0, - min_vortex_version: Some("0.36.0"), + min_library_version: Some("0.36.0"), }, added: &[ EditionMember::array(&"fastlanes.bitpacked"), diff --git a/vortex/src/editions/core/v2025_06.rs b/vortex-edition/src/declarations/core/v2025_06.rs similarity index 79% rename from vortex/src/editions/core/v2025_06.rs rename to vortex-edition/src/declarations/core/v2025_06.rs index 42cb7c04e6f..3d033d280ed 100644 --- a/vortex/src/editions/core/v2025_06.rs +++ b/vortex-edition/src/declarations/core/v2025_06.rs @@ -3,10 +3,10 @@ //! The `core` edition adding stable encodings released through June 2025. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; -use vortex_edition::EditionMember; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; +use crate::EditionMember; /// The June 2025 edition of the `core` family. pub const CORE_2025_06_0: EditionId = EditionId::new("core", 2025, 6, 0); @@ -15,7 +15,7 @@ pub const CORE_2025_06_0: EditionId = EditionId::new("core", 2025, 6, 0); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: CORE_2025_06_0, - min_vortex_version: Some("0.40.0"), + min_library_version: Some("0.40.0"), }, added: &[ EditionMember::array(&"vortex.pco"), diff --git a/vortex/src/editions/core/v2025_10.rs b/vortex-edition/src/declarations/core/v2025_10.rs similarity index 80% rename from vortex/src/editions/core/v2025_10.rs rename to vortex-edition/src/declarations/core/v2025_10.rs index fed71aee7e0..cfe2a79f0c3 100644 --- a/vortex/src/editions/core/v2025_10.rs +++ b/vortex-edition/src/declarations/core/v2025_10.rs @@ -3,10 +3,10 @@ //! The `core` edition adding stable encodings released through October 2025. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; -use vortex_edition::EditionMember; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; +use crate::EditionMember; /// The October 2025 edition of the `core` family. pub const CORE_2025_10_0: EditionId = EditionId::new("core", 2025, 10, 0); @@ -15,7 +15,7 @@ pub const CORE_2025_10_0: EditionId = EditionId::new("core", 2025, 10, 0); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: CORE_2025_10_0, - min_vortex_version: Some("0.54.0"), + min_library_version: Some("0.54.0"), }, added: &[ EditionMember::array(&"fastlanes.rle"), diff --git a/vortex/src/editions/core/v2026_08.rs b/vortex-edition/src/declarations/core/v2026_08.rs similarity index 88% rename from vortex/src/editions/core/v2026_08.rs rename to vortex-edition/src/declarations/core/v2026_08.rs index 07979c9976f..a9022d8d569 100644 --- a/vortex/src/editions/core/v2026_08.rs +++ b/vortex-edition/src/declarations/core/v2026_08.rs @@ -3,10 +3,10 @@ //! The frozen August 2026 core editions. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; -use vortex_edition::EditionMember; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; +use crate::EditionMember; /// The August 2026 core edition containing zoned layouts. pub const CORE_2026_08_0: EditionId = EditionId::new("core", 2026, 8, 0); @@ -23,7 +23,7 @@ pub const CORE_2026_08_0: EditionId = EditionId::new("core", 2026, 8, 0); pub static DECLARATION_0: EditionDeclaration = EditionDeclaration { edition: Edition { id: CORE_2026_08_0, - min_vortex_version: Some("0.84.0"), + min_library_version: Some("0.84.0"), }, added: &[ EditionMember::layout(&"vortex.zoned"), @@ -43,7 +43,7 @@ pub const CORE_2026_08_1: EditionId = EditionId::new("core", 2026, 8, 1); pub static DECLARATION_1: EditionDeclaration = EditionDeclaration { edition: Edition { id: CORE_2026_08_1, - min_vortex_version: Some("0.84.0"), + min_library_version: Some("0.84.0"), }, added: &[EditionMember::array(&"vortex.onpair")], }; diff --git a/vortex/src/editions/core/v2026_08_2.rs b/vortex-edition/src/declarations/core/v2026_08_2.rs similarity index 67% rename from vortex/src/editions/core/v2026_08_2.rs rename to vortex-edition/src/declarations/core/v2026_08_2.rs index 6c423b2f838..86222d23724 100644 --- a/vortex/src/editions/core/v2026_08_2.rs +++ b/vortex-edition/src/declarations/core/v2026_08_2.rs @@ -1,12 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! The August 2026 draft core edition adding canonical Map arrays. +//! The August 2026 core edition adding canonical Map arrays. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; -use vortex_edition::EditionMember; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; +use crate::EditionMember; /// The third August 2026 edition of the `core` family. pub const CORE_2026_08_2: EditionId = EditionId::new("core", 2026, 8, 2); @@ -15,7 +15,7 @@ pub const CORE_2026_08_2: EditionId = EditionId::new("core", 2026, 8, 2); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: CORE_2026_08_2, - min_vortex_version: None, + min_library_version: Some("0.85.0"), }, added: &[EditionMember::array(&"vortex.map")], }; diff --git a/vortex/src/editions/core/v2026_08_3.rs b/vortex-edition/src/declarations/core/v2026_08_3.rs similarity index 70% rename from vortex/src/editions/core/v2026_08_3.rs rename to vortex-edition/src/declarations/core/v2026_08_3.rs index 20c3e092b77..f6f2bb1010b 100644 --- a/vortex/src/editions/core/v2026_08_3.rs +++ b/vortex-edition/src/declarations/core/v2026_08_3.rs @@ -1,12 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! The August 2026 draft core edition adding Variant arrays and UUID extension dtypes. +//! The August 2026 core edition adding Variant arrays and UUID extension dtypes. -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; -use vortex_edition::EditionMember; +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; +use crate::EditionMember; /// The fourth August 2026 edition of the `core` family. pub const CORE_2026_08_3: EditionId = EditionId::new("core", 2026, 8, 3); @@ -15,7 +15,7 @@ pub const CORE_2026_08_3: EditionId = EditionId::new("core", 2026, 8, 3); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: CORE_2026_08_3, - min_vortex_version: None, + min_library_version: Some("0.85.0"), }, added: &[ EditionMember::array(&"vortex.parquet.variant"), diff --git a/vortex-edition/src/declarations/mod.rs b/vortex-edition/src/declarations/mod.rs new file mode 100644 index 00000000000..39132db9283 --- /dev/null +++ b/vortex-edition/src/declarations/mod.rs @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The first-party Vortex edition declarations, one module per edition. +//! +//! These are plain constants naming components by id, so they depend on nothing but the types +//! in this crate. That keeps them cheap to read: tooling that only needs to know what an +//! edition contains — `cargo run -p xtask -- generate-editions`, for one — can depend on +//! this crate alone rather than on the whole of `vortex`. +//! +//! The `vortex` facade re-exports everything here and owns the session wiring: registering +//! the declarations and selecting which of them the default writer may emit. + +pub mod core; +pub mod preview; + +use crate::EditionDeclaration; +use crate::EditionFamily; + +/// The first-party edition families. Every family must be declared before its editions. +pub static EDITION_FAMILIES: &[&EditionFamily] = &[&core::FAMILY, &preview::FAMILY]; + +/// The first-party Vortex edition declarations. +pub static EDITION_DECLARATIONS: &[&EditionDeclaration] = &[ + &core::v2025_05::DECLARATION, + &core::v2025_06::DECLARATION, + &core::v2025_10::DECLARATION, + &core::v2026_08::DECLARATION_0, + &core::v2026_08::DECLARATION_1, + &core::v2026_08_2::DECLARATION, + &core::v2026_08_3::DECLARATION, + &preview::v2026_08::DECLARATION, +]; diff --git a/vortex-edition/src/declarations/preview/mod.rs b/vortex-edition/src/declarations/preview/mod.rs new file mode 100644 index 00000000000..e203e35a475 --- /dev/null +++ b/vortex-edition/src/declarations/preview/mod.rs @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The `preview` edition family: additive, opt-in components awaiting adoption into `core`. + +use crate::EditionFamily; + +/// The `preview` family: stable opt-in components not yet available by default. +pub static FAMILY: EditionFamily = EditionFamily { + name: "preview", + origin: "vortex", + doc: "Additive, opt-in components maintained as part of Vortex but not yet adopted by the \ +default core writer. Components enter preview only once their serialized contracts are ready for \ +broad testing; independently evolving work remains in its own family until then.", +}; + +pub mod v2026_08; + +pub use v2026_08::PREVIEW_2026_08_0; diff --git a/vortex-edition/src/declarations/preview/v2026_08.rs b/vortex-edition/src/declarations/preview/v2026_08.rs new file mode 100644 index 00000000000..94396ecf162 --- /dev/null +++ b/vortex-edition/src/declarations/preview/v2026_08.rs @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The August 2026 `preview` edition. + +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; + +/// The August 2026 draft edition of the `preview` family. +pub const PREVIEW_2026_08_0: EditionId = EditionId::new("preview", 2026, 8, 0); + +/// The empty declaration of [`PREVIEW_2026_08_0`]. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: PREVIEW_2026_08_0, + min_library_version: None, + }, + added: &[], +}; diff --git a/vortex-edition/src/lib.rs b/vortex-edition/src/lib.rs index 6e5c609a60a..24fa0a40e64 100644 --- a/vortex-edition/src/lib.rs +++ b/vortex-edition/src/lib.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! Definitions of Vortex *editions*: named, frozen sets of components that a writer may put -//! in a file, carrying a forever read-compatibility guarantee. +//! Definitions of Vortex *editions*: named sets of serialized component IDs. Frozen editions +//! carry a forever read-compatibility guarantee; draft editions do not. //! //! Editions live on the session, like encodings do: [`EditionSession`] holds the registered //! editions and [`EnabledEditions`] selects which of them a writer may emit. Declarations @@ -13,20 +13,25 @@ //! //! Every membership is typed by a [`ComponentKind`], and members are resolved one kind at a //! time with [`EditionSessionExt::enabled_component_ids`]: the file writer restricts the -//! arrays, layouts, extension dtypes, and aggregates it writes from separate id sets, never one -//! untyped set. +//! arrays, layouts, extension dtypes, and aggregates it writes from separate id sets. Array +//! memberships name wire IDs rather than in-memory array representations. An array plugin may +//! serialize one current in-memory representation under several historical IDs. The serialization +//! context validates the ID chosen by the plugin before writing it. Readers resolve the ID stored +//! in the file and either deserialize it into the current representation or reject it as unknown. //! -//! An edition is a **draft** until its [`Edition::min_vortex_version`] is recorded — -//! recording it is the act of freezing. The per-edition member sets are computed from the -//! registered declarations by [`EditionSession::components_in`], and correctness is enforced -//! by unit tests: [`EditionSession::validate`] checks a whole registry, and -//! [`test_harness::validate_edition`] validates one edition's constraints — call it once in +//! An edition is represented as a **draft** until its [`Edition::min_library_version`] is +//! recorded. A stable edition may freeze in the release that cuts it; once that release version +//! is known, the field is backfilled to document the freeze. The per-edition member sets are +//! computed from the registered declarations by [`EditionSession::components_in`], and +//! correctness is enforced by unit tests: [`EditionSession::validate`] checks a whole registry, +//! and [`test_harness::validate_edition`] validates one edition's constraints — call it once in //! the `#[cfg(test)]` module of each edition definition. //! -//! The first-party edition declarations live in the public `vortex` crate, which registers -//! and enables them on the default session. See the published spec at +//! The first-party edition declarations live in this crate. The public `vortex` crate +//! re-exports them and registers and enables them on the default session. See the published spec at //! . +pub mod declarations; mod session; pub mod test_harness; #[cfg(test)] @@ -38,6 +43,8 @@ use std::fmt::Debug; use std::fmt::Display; use std::fmt::Formatter; +pub use declarations::EDITION_DECLARATIONS; +pub use declarations::EDITION_FAMILIES; pub use session::EditionSession; pub use session::EditionSessionExt; pub use session::EnabledEditions; @@ -45,18 +52,19 @@ use vortex_session::registry::Id; /// The identifier of an edition, e.g. `core2026.07.0`. /// -/// The `family` names an independently versioned, additive group of components (`core` is the -/// set the default writer emits). The date components record when the edition was frozen and -/// order editions chronologically *within* a family; there is no ordering across families. +/// The `family` names an independently versioned, additive group of members (`core` is the set +/// available to the default writer). For `core`, the date components record when the edition +/// freezes; that date is prospective while the edition is still a draft. Dates order editions +/// chronologically *within* a family; there is no ordering across families. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct EditionId { /// The edition family, e.g. `core`. pub family: &'static str, - /// Year the edition was cut. + /// Year in the edition date. For `core`, this is the freeze year. pub year: u16, - /// Month the edition was cut. + /// Month in the edition date. For `core`, this is the freeze month. pub month: u8, - /// Distinguishes editions cut in the same month; normally `0`. + /// Distinguishes editions with the same family, year, and month; normally `0`. pub version: u8, } @@ -114,7 +122,51 @@ impl Display for EditionId { } } -/// The kind of component an edition membership covers. +/// A family of editions: an independently versioned, additive group of members, registered +/// with [`EditionSession::declare_family`]. +/// +/// Every [`EditionId`] names one. Declaring the family is what makes the name real: +/// [`EditionSession::validate`] rejects an edition whose family was never declared, so a typo +/// cannot quietly mint a family of one. +#[derive(Clone, Copy, Debug)] +pub struct EditionFamily { + /// The family name, matching the [`EditionId::family`] of its editions, e.g. `core`. + pub name: &'static str, + /// The library or project whose releases provide readers for this family's editions. + /// [`Edition::min_library_version`] refers to versions of this origin. + pub origin: &'static str, + /// What the family is for. Exported into the family's record, so a few sentences at + /// most: the long form belongs in the published spec. + pub doc: &'static str, +} + +impl EditionFamily { + /// Validate the family's form: a non-empty lowercase name, origin, and doc. Checked for every + /// declared family by [`EditionSession::validate`]. + pub fn validate(&self) -> Result<(), EditionError> { + if self.name.is_empty() || !self.name.chars().all(|c| c.is_ascii_lowercase()) { + return Err(EditionError::new(format!( + "edition family {:?} must have a non-empty lowercase name, e.g. `core`", + self.name + ))); + } + if self.origin.trim().is_empty() { + return Err(EditionError::new(format!( + "edition family {} must name its origin library or project", + self.name + ))); + } + if self.doc.trim().is_empty() { + return Err(EditionError::new(format!( + "edition family {} must document what it is for", + self.name + ))); + } + Ok(()) + } +} + +/// The kind of member an edition membership covers. /// /// Ids are unique per kind, not globally: a layout named `vortex.flat` and an array named /// `vortex.flat` are different members. Every membership records its kind, and the writer @@ -122,7 +174,8 @@ impl Display for EditionId { /// written layouts. Further kinds (scalar functions, say) can be added the same way. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum ComponentKind { - /// An array encoding, e.g. `vortex.alp`, registered in the session's array registry. + /// A serialized array representation, e.g. `vortex.alp`, registered in the session's array + /// registry. Array, /// A layout encoding, e.g. `vortex.flat`, registered in the session's layout registry. Layout, @@ -144,32 +197,39 @@ impl Display for ComponentKind { } } -/// An edition: a named set of components with a read-compatibility guarantee, registered with -/// [`EditionSession::declare_edition`]. The set itself is computed from the registered -/// [`EditionInclusion`]s by [`EditionSession::components_in`]. +/// An edition: a named set of serialized components that can acquire a read-compatibility +/// guarantee, registered with [`EditionSession::declare_edition`]. +/// The set itself is computed from the registered [`EditionInclusion`]s by +/// [`EditionSession::components_in`]. #[derive(Clone, Copy, Debug)] pub struct Edition { - /// The edition identifier. Also carries the freeze date: `core2026.07.0` freezes in - /// 2026-07. + /// The edition identifier. For a `core` edition, its date records when it freezes. pub id: EditionId, - /// The minimum Vortex version whose reader supports every member of this edition. + /// The minimum version of the edition family's [`EditionFamily::origin`] whose reader + /// supports every member of this edition. /// - /// Recording this is the act of freezing: an edition with `None` is a **draft** — being - /// assembled, carrying no guarantee, free to change, never the default write target. + /// A stable edition may freeze in the release that cuts it. Until that release is cut, its + /// version is not known and this remains `None`. The version is then backfilled to document + /// the already completed freeze and identify the first released reader supporting every + /// member. A draft has no recorded read-forever guarantee; that does not imply that its + /// behavior is expected to change. /// Validated against the members' [`EditionInclusion::required_vortex_release`] values: /// no member may require a version newer than the edition declares. - pub min_vortex_version: Option<&'static str>, + pub min_library_version: Option<&'static str>, } impl Edition { - /// A draft is an edition whose `min_vortex_version` has not been recorded yet. + /// A draft is an edition whose `min_library_version` has not been recorded yet. + /// + /// This describes the absence of a frozen compatibility guarantee, not necessarily the + /// implementation stability of its members. pub fn is_draft(&self) -> bool { - self.min_vortex_version.is_none() + self.min_library_version.is_none() } } -/// Declares that a component is a member of an edition — and of every later edition of the -/// same family. Registered with [`EditionSession::declare_inclusion`]. +/// Declares that a serialized component is a member of an edition — and of every later edition of +/// the same family. Registered with [`EditionSession::declare_inclusion`]. #[derive(Clone, Copy, Debug)] pub struct EditionInclusion { /// What the membership covers. Ids are unique per kind, so this is part of the @@ -179,8 +239,8 @@ pub struct EditionInclusion { pub component_id: Id, /// The first edition this component is a member of. pub since: EditionId, - /// The earliest Vortex release able to read and execute this component, recorded from - /// evidence (e.g. compat-fixture history). `None` until recorded. + /// The earliest Vortex release supporting this member, recorded from evidence (e.g. + /// compat-fixture history for serialized components). `None` until recorded. pub required_vortex_release: Option<&'static str>, } @@ -219,14 +279,14 @@ impl AsComponentId for &'static str { } } -/// A component that joins an edition, named by id string or vtable and tagged with the kind -/// of registry it belongs to. Built with the per-kind constructors, so a declaration reads +/// A member that joins an edition, named by id string or vtable and tagged with its kind. +/// Built with the per-kind constructors, so a declaration reads /// as `EditionMember::array(&"vortex.alp")`. #[derive(Clone, Copy, Debug)] pub struct EditionMember { - /// What kind of component this is. + /// What kind of member this is. pub kind: ComponentKind, - /// The component, named by id string or by vtable. + /// The member, named by id string or by vtable. pub component: &'static dyn AsComponentId, } @@ -264,15 +324,15 @@ impl EditionMember { } } -/// Declares an edition together with the components that join the family at it, in one -/// block. Registered with [`EditionSession::declare`], which derives each member's -/// membership (`since` = the declared edition) from the block structure. +/// Declares an edition together with its new members in one block. Registered with +/// [`EditionSession::declare`], which derives each entry's membership (`since` = the declared +/// edition) from the block structure. #[derive(Clone, Copy, Debug)] pub struct EditionDeclaration { /// The edition being declared. pub edition: Edition, - /// The components that join the family at this edition, each tagged with its - /// [`ComponentKind`]. Members of earlier editions are inherited and never restated. + /// The members that join the family at this edition, each tagged with its [`ComponentKind`]. + /// Earlier entries are inherited and never restated. pub added: &'static [EditionMember], } diff --git a/vortex-edition/src/session.rs b/vortex-edition/src/session.rs index b428062c17c..a80327726df 100644 --- a/vortex-edition/src/session.rs +++ b/vortex-edition/src/session.rs @@ -18,6 +18,7 @@ use crate::ComponentKind; use crate::Edition; use crate::EditionDeclaration; use crate::EditionError; +use crate::EditionFamily; use crate::EditionId; use crate::EditionInclusion; use crate::parse_release; @@ -36,12 +37,14 @@ pub struct EditionSession { #[derive(Debug, Default)] struct Inner { + /// Keyed by family name. + families: BTreeMap, /// Keyed by the display form of the edition id. editions: BTreeMap, - /// One map per component kind, each keyed by interned component id, because ids are - /// only unique within a kind. Resolving a kind scans that kind's map alone, never the - /// other kinds' entries. Ordered by kind, then by the id's string form. - inclusions: BTreeMap>, + /// One map per member kind, each keyed by interned member id, because ids are only unique + /// within a kind. An id may have one inclusion per family. Ordered by kind, then by the id's + /// string form. + inclusions: BTreeMap>>, } /// Registry of enabled editions, keyed by interned edition family. @@ -82,9 +85,8 @@ impl EditionSession { } } - /// Declare an edition together with the components that join the family at it. Each - /// added member's membership (`since`) is the declared edition; members of earlier - /// editions are inherited and must not be restated. + /// Declare an edition together with the members added at it. Each entry's membership + /// (`since`) is the declared edition; earlier entries are inherited and must not be restated. pub fn declare(&self, declaration: &EditionDeclaration) -> Result<(), EditionError> { self.declare_edition(declaration.edition)?; for member in declaration.added { @@ -97,6 +99,31 @@ impl EditionSession { Ok(()) } + /// Declare an edition family. Errors if a family with the same name is already + /// declared. Every family must be declared before [`EditionSession::validate`] will + /// accept editions belonging to it. + pub fn declare_family(&self, family: &EditionFamily) -> Result<(), EditionError> { + let mut inner = self.inner.write(); + if inner.families.contains_key(family.name) { + return Err(EditionError::new(format!( + "duplicate edition family {}", + family.name + ))); + } + inner.families.insert(family.name.to_string(), *family); + Ok(()) + } + + /// All declared families, sorted by name. + pub fn families(&self) -> Vec { + self.inner.read().families.values().copied().collect() + } + + /// Find a declared family by name. + pub fn find_family(&self, name: &str) -> Option { + self.inner.read().families.get(name).copied() + } + /// Declare an edition. Errors if an edition with the same id is already declared. pub fn declare_edition(&self, edition: Edition) -> Result<(), EditionError> { let mut inner = self.inner.write(); @@ -108,19 +135,39 @@ impl EditionSession { Ok(()) } - /// Declare an edition inclusion. Errors if the component already has one: a component - /// belongs to exactly one family, with one membership interval. Kind is part of the + /// Declare an edition inclusion. A component may belong to multiple families but joins each + /// family only once. A newer wire representation uses a new component ID. Kind is part of the /// key, so an array encoding and a layout may share an id. pub fn declare_inclusion(&self, inclusion: EditionInclusion) -> Result<(), EditionError> { let mut inner = self.inner.write(); let by_id = inner.inclusions.entry(inclusion.kind).or_default(); - if by_id.contains_key(&inclusion.component_id) { + let history = by_id.entry(inclusion.component_id).or_default(); + let previous = history + .iter() + .filter(|existing| existing.since.family == inclusion.since.family) + .max_by_key(|existing| { + ( + existing.since.year, + existing.since.month, + existing.since.version, + ) + }); + + if let Some(previous) = previous { return Err(EditionError::new(format!( - "duplicate edition inclusion for {} {}", - inclusion.kind, inclusion.component_id + "{} {} already joined family {} in edition {}", + inclusion.kind, inclusion.component_id, inclusion.since.family, previous.since, ))); } - by_id.insert(inclusion.component_id, inclusion); + history.push(inclusion); + history.sort_by_key(|entry| { + ( + entry.since.family, + entry.since.year, + entry.since.month, + entry.since.version, + ) + }); Ok(()) } @@ -145,9 +192,9 @@ impl EditionSession { .rfind(|e| e.id.family == family && !e.is_draft()) } - /// Compute an edition's members of one kind: every declared inclusion of that kind in - /// the edition's family whose `since` is at or before it, sorted by component id. Only - /// that kind's declarations are scanned. + /// Compute an edition's members of one kind, sorted by component id. For each id, this returns + /// its inclusion in the edition's family when it joined at or before the requested edition. + /// Only that kind's declarations are scanned. pub fn components_in(&self, edition: &EditionId, kind: ComponentKind) -> Vec { let inner = self.inner.read(); let Some(by_id) = inner.inclusions.get(&kind) else { @@ -155,25 +202,47 @@ impl EditionSession { }; by_id .values() - .filter(|inclusion| inclusion.since.is_at_or_before(edition)) - .copied() + .filter_map(|history| { + history + .iter() + .filter(|inclusion| inclusion.since.is_at_or_before(edition)) + .max_by_key(|inclusion| { + ( + inclusion.since.year, + inclusion.since.month, + inclusion.since.version, + ) + }) + .copied() + }) .collect() } - /// Validate all registered declarations. Errors on inclusions referencing undeclared - /// editions, editions out of chronological order within a family (unversioned drafts - /// must be newest), malformed version strings, and members requiring a release newer - /// than their edition declares. + /// Validate all registered declarations. Errors on editions in undeclared families, + /// inclusions referencing undeclared editions, editions out of chronological order within + /// a family (unversioned drafts must be newest), malformed version strings, and members + /// requiring a release newer than their edition declares. pub fn validate(&self) -> Result<(), EditionError> { let editions = self.editions(); + for family in self.families() { + family.validate()?; + } + for edition in &editions { edition.id.validate()?; - if let Some(version) = edition.min_vortex_version + if self.find_family(edition.id.family).is_none() { + return Err(EditionError::new(format!( + "edition {} belongs to undeclared family {}; declare the family before \ + its editions", + edition.id, edition.id.family, + ))); + } + if let Some(version) = edition.min_library_version && parse_release(version).is_none() { return Err(EditionError::new(format!( - "edition {} declares malformed min_vortex_version {version:?}", + "edition {} declares malformed min_library_version {version:?}", edition.id ))); } @@ -192,7 +261,12 @@ impl EditionSession { } let inner = self.inner.read(); - for inclusion in inner.inclusions.values().flat_map(|by_id| by_id.values()) { + for inclusion in inner + .inclusions + .values() + .flat_map(|by_id| by_id.values()) + .flatten() + { inclusion.validate()?; let Some(edition) = inner.editions.get(&inclusion.since.to_string()) else { @@ -203,12 +277,12 @@ impl EditionSession { }; if let Some(required) = inclusion.required_vortex_release.and_then(parse_release) - && let Some(declared) = edition.min_vortex_version.and_then(parse_release) + && let Some(declared) = edition.min_library_version.and_then(parse_release) && required > declared { return Err(EditionError::new(format!( "{} {} requires release {}, newer than edition {}'s declared \ - min_vortex_version", + min_library_version", inclusion.kind, inclusion.component_id, inclusion.required_vortex_release.unwrap_or_default(), @@ -276,8 +350,8 @@ pub trait EditionSessionExt: SessionExt { Ok(()) } - /// Resolve the ids of one [`ComponentKind`] across all enabled editions: what a writer - /// may emit for that kind. + /// Resolve the ids of one [`ComponentKind`] across all enabled editions: what a writer may + /// emit for that kind. /// /// Ids are only unique within a kind, so this never mixes kinds. An empty result means the /// enabled editions permit no components of this kind. diff --git a/vortex-edition/src/tests.rs b/vortex-edition/src/tests.rs index 21a45bd2db8..4e0d3846682 100644 --- a/vortex-edition/src/tests.rs +++ b/vortex-edition/src/tests.rs @@ -6,6 +6,7 @@ use vortex_session::VortexSession; use crate::ComponentKind; use crate::Edition; use crate::EditionDeclaration; +use crate::EditionFamily; use crate::EditionId; use crate::EditionInclusion; use crate::EditionMember; @@ -13,6 +14,18 @@ use crate::EditionSession; use crate::EditionSessionExt; use crate::EnabledEditions; +static TEST_FAMILY: EditionFamily = EditionFamily { + name: "test", + origin: "vortex-edition-tests", + doc: "A family used by the unit tests.", +}; + +static OTHER_FAMILY: EditionFamily = EditionFamily { + name: "other", + origin: "vortex-edition-tests", + doc: "A second family, for checking that families stay independent.", +}; + const FIRST: EditionId = EditionId::new("test", 2026, 1, 0); const SECOND: EditionId = EditionId::new("test", 2026, 7, 0); @@ -20,7 +33,7 @@ static DECLARATIONS: &[EditionDeclaration] = &[ EditionDeclaration { edition: Edition { id: FIRST, - min_vortex_version: None, + min_library_version: None, }, added: &[ EditionMember::array(&"test.alpha"), @@ -30,14 +43,20 @@ static DECLARATIONS: &[EditionDeclaration] = &[ EditionDeclaration { edition: Edition { id: SECOND, - min_vortex_version: None, + min_library_version: None, }, - added: &[EditionMember::array(&"test.gamma")], + added: &[ + EditionMember::array(&"test.alpha_v2"), + EditionMember::array(&"test.gamma"), + ], }, ]; fn session() -> EditionSession { let editions = EditionSession::empty(); + editions + .declare_family(&TEST_FAMILY) + .unwrap_or_else(|e| panic!("declaring the test family: {e}")); for declaration in DECLARATIONS { editions .declare(declaration) @@ -58,24 +77,36 @@ fn editions_pass_the_test_harness() -> Result<(), crate::EditionError> { } #[test] -fn membership_is_transitive() { +fn membership_is_transitive() -> Result<(), crate::EditionError> { let editions = session(); let first = editions.components_in(&FIRST, ComponentKind::Array); let ids: Vec<&str> = first.iter().map(|i| i.component_id.as_str()).collect(); assert_eq!(ids, ["test.alpha", "test.beta"]); - // Members of the first edition are members of the second by inheritance, with their - // `since` still recording the edition they actually joined in. + // Members of the first edition are inherited. A newer wire representation has its own ID, so + // both the historical and current representations remain explicit members. let second = editions.components_in(&SECOND, ComponentKind::Array); let ids: Vec<&str> = second.iter().map(|i| i.component_id.as_str()).collect(); - assert_eq!(ids, ["test.alpha", "test.beta", "test.gamma"]); - assert!( - second - .iter() - .filter(|i| i.component_id.as_str() != "test.gamma") - .all(|i| i.since == FIRST) + assert_eq!( + ids, + ["test.alpha", "test.alpha_v2", "test.beta", "test.gamma"] ); + let alpha = second + .iter() + .find(|i| i.component_id.as_str() == "test.alpha") + .ok_or_else(|| crate::EditionError::new("test.alpha is a member"))?; + assert_eq!(alpha.since, FIRST); + let alpha_v2 = second + .iter() + .find(|i| i.component_id.as_str() == "test.alpha_v2") + .ok_or_else(|| crate::EditionError::new("test.alpha_v2 is a member"))?; + assert_eq!(alpha_v2.since, SECOND); + let beta = second + .iter() + .find(|i| i.component_id.as_str() == "test.beta") + .ok_or_else(|| crate::EditionError::new("test.beta is a member"))?; + assert_eq!(beta.since, FIRST); // The second edition's delta is exactly the members declared at it. let added: Vec<&str> = second @@ -83,7 +114,7 @@ fn membership_is_transitive() { .filter(|i| i.since == SECOND) .map(|i| i.component_id.as_str()) .collect(); - assert_eq!(added, ["test.gamma"]); + assert_eq!(added, ["test.alpha_v2", "test.gamma"]); // Inheritance never flows backwards, extends to later editions of the family, and // never crosses families. @@ -91,7 +122,7 @@ fn membership_is_transitive() { let third = EditionId::new("test", 2026, 10, 0); assert_eq!( editions.components_in(&third, ComponentKind::Array).len(), - 3 + 4 ); let other = EditionId::new("other", 2026, 10, 0); assert!( @@ -99,6 +130,7 @@ fn membership_is_transitive() { .components_in(&other, ComponentKind::Array) .is_empty() ); + Ok(()) } #[test] @@ -110,16 +142,17 @@ fn drafts_and_current() { // Freezing the first edition makes it current; the second stays a draft. let editions = EditionSession::empty(); + editions.declare_family(&TEST_FAMILY).unwrap(); editions .declare_edition(Edition { id: FIRST, - min_vortex_version: Some("0.60.0"), + min_library_version: Some("0.60.0"), }) .unwrap(); editions .declare_edition(Edition { id: SECOND, - min_vortex_version: None, + min_library_version: None, }) .unwrap(); assert!(editions.validate().is_ok()); @@ -161,10 +194,9 @@ fn registered_and_enabled_editions_are_separate() -> Result<(), crate::EditionEr .collect::>(), ["test.alpha", "test.beta"] ); - session.enable_edition(SECOND)?; assert_eq!(session.enabled_editions().editions(), [SECOND]); - assert_eq!(session.enabled_component_ids(ComponentKind::Array).len(), 3); + assert_eq!(session.enabled_component_ids(ComponentKind::Array).len(), 4); // Selecting an older edition in the same family replaces the newer one and removes // encodings that joined after it. @@ -189,14 +221,17 @@ fn enabled_editions_are_independent_across_families() -> Result<(), crate::Editi static OTHER_DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: OTHER, - min_vortex_version: None, + min_library_version: None, }, added: &[EditionMember::array(&"other.delta")], }; let session = VortexSession::empty().with::(); + session.editions().declare_family(&TEST_FAMILY)?; + session.editions().declare_family(&OTHER_FAMILY)?; session.register_edition(&DECLARATIONS[0])?; session.register_edition(&OTHER_DECLARATION)?; + session.editions().validate()?; session.enable_edition(FIRST)?; session.enable_edition(OTHER)?; @@ -207,6 +242,37 @@ fn enabled_editions_are_independent_across_families() -> Result<(), crate::Editi Ok(()) } +#[test] +fn serialized_array_ids_can_be_added_by_an_opt_in_family() -> Result<(), crate::EditionError> { + const OPT_IN: EditionId = EditionId::new("other", 2026, 8, 0); + static OPT_IN_DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: OPT_IN, + min_library_version: None, + }, + added: &[EditionMember::array(&"test.alpha_v2")], + }; + + let session = VortexSession::empty().with::(); + session.editions().declare_family(&TEST_FAMILY)?; + session.editions().declare_family(&OTHER_FAMILY)?; + session.register_edition(&DECLARATIONS[0])?; + session.register_edition(&OPT_IN_DECLARATION)?; + session.editions().validate()?; + session.enable_edition(FIRST)?; + session.enable_edition(OPT_IN)?; + + assert_eq!( + session + .enabled_component_ids(ComponentKind::Array) + .iter() + .map(|id| id.as_str()) + .collect::>(), + ["test.alpha", "test.alpha_v2", "test.beta"] + ); + Ok(()) +} + #[test] fn duplicate_declarations_error() { let editions = session(); @@ -214,7 +280,7 @@ fn duplicate_declarations_error() { editions .declare_edition(Edition { id: FIRST, - min_vortex_version: None, + min_library_version: None, }) .is_err() ); @@ -236,7 +302,7 @@ fn validate_rejects_inconsistent_declarations() -> Result<(), crate::EditionErro let editions = EditionSession::empty(); editions.declare_edition(Edition { id: FIRST, - min_vortex_version: Some("0.70.0"), + min_library_version: Some("0.70.0"), })?; editions.declare_inclusion(EditionInclusion { required_vortex_release: Some("0.80.0"), @@ -248,11 +314,11 @@ fn validate_rejects_inconsistent_declarations() -> Result<(), crate::EditionErro let editions = EditionSession::empty(); editions.declare_edition(Edition { id: FIRST, - min_vortex_version: None, + min_library_version: None, })?; editions.declare_edition(Edition { id: SECOND, - min_vortex_version: Some("0.70.0"), + min_library_version: Some("0.70.0"), })?; assert!(editions.validate().is_err()); @@ -260,7 +326,7 @@ fn validate_rejects_inconsistent_declarations() -> Result<(), crate::EditionErro let editions = EditionSession::empty(); editions.declare_edition(Edition { id: EditionId::new("Test", 2026, 13, 0), - min_vortex_version: None, + min_library_version: None, })?; assert!(editions.validate().is_err()); @@ -268,7 +334,7 @@ fn validate_rejects_inconsistent_declarations() -> Result<(), crate::EditionErro let editions = EditionSession::empty(); editions.declare_edition(Edition { id: FIRST, - min_vortex_version: None, + min_library_version: None, })?; editions.declare_inclusion(EditionInclusion::array("Test.ALPHA", FIRST))?; assert!(editions.validate().is_err()); @@ -291,13 +357,56 @@ fn edition_id_display() { assert_eq!(FIRST.to_string(), "test2026.01.0"); } +#[test] +fn families_must_be_declared_before_their_editions() -> Result<(), crate::EditionError> { + // An edition whose family was never declared: the name would otherwise be whatever the + // declaration happened to spell, and a typo would mint a family of one. + let editions = EditionSession::empty(); + editions.declare(&DECLARATIONS[0])?; + assert!(editions.validate().is_err()); + + editions.declare_family(&TEST_FAMILY)?; + editions.validate()?; + + // Declaring the same family twice is an error, as it is for editions. + assert!(editions.declare_family(&TEST_FAMILY).is_err()); + Ok(()) +} + +#[test] +fn families_must_document_themselves() { + let editions = EditionSession::empty(); + editions + .declare_family(&EditionFamily { + name: "undocumented", + origin: "vortex-edition-tests", + doc: " ", + }) + .unwrap(); + assert!(editions.validate().is_err()); +} + +#[test] +fn families_must_name_their_origin() { + let editions = EditionSession::empty(); + editions + .declare_family(&EditionFamily { + name: "unowned", + origin: " ", + doc: "A family without an origin.", + }) + .unwrap(); + let error = editions.validate().unwrap_err(); + assert!(error.to_string().contains("origin library or project")); +} + #[test] fn kinds_are_resolved_independently() -> Result<(), crate::EditionError> { // `test.alpha` is declared under both kinds: same id, two distinct members. static MIXED: EditionDeclaration = EditionDeclaration { edition: Edition { id: FIRST, - min_vortex_version: None, + min_library_version: None, }, added: &[ EditionMember::array(&"test.alpha"), diff --git a/vortex-file/benches/split_collection.rs b/vortex-file/benches/split_collection.rs index 4811e84c82d..2c4ccd0053e 100644 --- a/vortex-file/benches/split_collection.rs +++ b/vortex-file/benches/split_collection.rs @@ -21,15 +21,8 @@ use vortex_array::arrays::ChunkedArray; use vortex_array::arrays::StructArray; use vortex_array::dtype::Field; use vortex_array::dtype::FieldMask; -use vortex_array::dtype::session::DTypeSessionExt; -use vortex_array::session::ArraySessionExt; use vortex_buffer::Buffer; use vortex_buffer::ByteBufferMut; -use vortex_edition::ComponentKind; -use vortex_edition::Edition; -use vortex_edition::EditionId; -use vortex_edition::EditionInclusion; -use vortex_edition::EditionSessionExt; use vortex_file::OpenOptionsSessionExt; use vortex_file::VortexFile; use vortex_file::WriteOptionsSessionExt; @@ -41,7 +34,6 @@ use vortex_layout::layouts::repartition::RepartitionStrategy; use vortex_layout::layouts::repartition::RepartitionWriterOptions; use vortex_layout::scan::split_by::SplitBy; use vortex_layout::session::LayoutSession; -use vortex_layout::session::LayoutSessionExt; use vortex_session::VortexSession; use vortex_utils::aliases::hash_map::HashMap; @@ -68,69 +60,9 @@ static SESSION: LazyLock = LazyLock::new(|| { .with::() .with_tokio(); vortex_file::register_default_encodings(&session); - enable_all_registered_array_encodings(&session); session }); -const BENCH_EDITION: EditionId = EditionId::new("bench", 2026, 8, 0); - -fn enable_all_registered_array_encodings(session: &VortexSession) { - let editions = session.editions(); - editions - .declare_edition(Edition { - id: BENCH_EDITION, - min_vortex_version: None, - }) - .unwrap(); - let component_ids = [ - ( - ComponentKind::Array, - session - .arrays() - .registry() - .read(|map| map.keys().copied().collect::>()), - ), - ( - ComponentKind::Layout, - session - .layouts() - .registry() - .read(|map| map.keys().copied().collect::>()), - ), - ( - ComponentKind::DType, - session - .dtypes() - .registry() - .read(|map| map.keys().copied().collect::>()), - ), - ]; - for (kind, ids) in component_ids { - for id in ids { - editions - .declare_inclusion(EditionInclusion::new(kind, &id, BENCH_EDITION)) - .unwrap(); - } - } - for id in [ - "vortex.bounded_max", - "vortex.bounded_min", - "vortex.max", - "vortex.min", - "vortex.nan_count", - "vortex.null_count", - ] { - editions - .declare_inclusion(EditionInclusion::new( - ComponentKind::Aggregate, - id, - BENCH_EDITION, - )) - .unwrap(); - } - session.enable_edition(BENCH_EDITION).unwrap(); -} - fn make_file(columns: usize, chunks: usize) -> VortexFile { let field_names = (0..columns).map(|c| format!("col_{c}")).collect::>(); let struct_chunks = (0..chunks) @@ -159,6 +91,7 @@ fn make_file(columns: usize, chunks: usize) -> VortexFile { .block_on( SESSION .write_options() + .disable_editions() .with_strategy(strategy) .write(&mut buf, array.to_array_stream()), ) @@ -225,6 +158,7 @@ fn make_misaligned_file(columns: usize, chunks: usize) -> VortexFile { .block_on( SESSION .write_options() + .disable_editions() .with_strategy(strategy.build()) .write(&mut buf, array.to_array_stream()), ) diff --git a/vortex-file/src/lib.rs b/vortex-file/src/lib.rs index d66aaa54a9b..317aade342a 100644 --- a/vortex-file/src/lib.rs +++ b/vortex-file/src/lib.rs @@ -172,14 +172,12 @@ pub fn register_default_encodings(session: &VortexSession) { vortex_fsst::initialize(session); vortex_onpair::initialize(session); vortex_zigzag::initialize(session); + #[cfg(feature = "zstd")] + vortex_zstd::initialize(session); { let arrays = session.arrays(); arrays.register(Pco); - #[cfg(feature = "zstd")] - arrays.register(vortex_zstd::Zstd); - #[cfg(all(feature = "zstd", feature = "unstable_encodings"))] - arrays.register(vortex_zstd::ZstdBuffers); if use_experimental_patches() { arrays.register(Patched); } @@ -215,7 +213,7 @@ pub(crate) fn enable_all_registered_array_encodings(session: &VortexSession) { editions .declare_edition(Edition { id: TEST_EDITION, - min_vortex_version: None, + min_library_version: None, }) .map_err(|error| vortex_err!("{error}")) .vortex_expect("test edition is valid"); diff --git a/vortex-file/src/strategy.rs b/vortex-file/src/strategy.rs index 9d4dbb90610..c8110fe88c3 100644 --- a/vortex-file/src/strategy.rs +++ b/vortex-file/src/strategy.rs @@ -6,14 +6,12 @@ use std::num::NonZeroUsize; use std::sync::Arc; -use vortex_array::ArrayId; use vortex_array::dtype::FieldPath; use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_btrblocks::SchemeExt; use vortex_btrblocks::schemes::integer::IntDictScheme; use vortex_error::VortexExpect; use vortex_layout::LayoutStrategy; -use vortex_layout::LayoutStrategyEncodingValidator; use vortex_layout::layouts::buffered::BufferedStrategy; use vortex_layout::layouts::chunked::writer::ChunkedLayoutStrategy; use vortex_layout::layouts::collect::CollectStrategy; @@ -29,7 +27,6 @@ use vortex_layout::layouts::table::use_experimental_list_layout; use vortex_layout::layouts::zoned::writer::ZonedLayoutOptions; use vortex_layout::layouts::zoned::writer::ZonedStrategy; use vortex_utils::aliases::hash_map::HashMap; -use vortex_utils::aliases::hash_set::HashSet; const ONE_MEG: u64 = 1 << 20; @@ -59,7 +56,6 @@ pub struct WriteStrategyBuilder { row_block_size: usize, data_block_target_bytes: Option, field_writers: HashMap>, - allow_encodings: Option>, flat_strategy: Option>, probe_compressor: Option>, /// Whether to write list fields using [`ListLayoutStrategy`]. @@ -77,7 +73,6 @@ impl Default for WriteStrategyBuilder { row_block_size: 8192, data_block_target_bytes: Some(ONE_MEG), field_writers: HashMap::new(), - allow_encodings: None, flat_strategy: None, probe_compressor: None, use_list_layout: use_experimental_list_layout(), @@ -128,17 +123,6 @@ impl WriteStrategyBuilder { self } - /// Override the allowed array encodings for file writing. - /// - /// The configured flat leaf strategy is wrapped in a [`LayoutStrategyEncodingValidator`] - /// that recursively checks every chunk before passing it to the leaf writer. [`build`](Self::build) - /// also restricts any [`BtrBlocksCompressorBuilder`] to these encodings, independent of the - /// order in which the builder and this policy were configured. - pub fn with_allow_encodings(mut self, allow_encodings: HashSet) -> Self { - self.allow_encodings = Some(allow_encodings); - self - } - /// Override the flat layout strategy used for leaf chunks. /// /// By default, this uses [`FlatLayoutStrategy`]. This can be used to substitute a custom @@ -150,9 +134,8 @@ impl WriteStrategyBuilder { /// Override the default [`BtrBlocksCompressorBuilder`] used for compression. /// - /// The builder is finalized during [`build`](Self::build), producing two compressors: one for - /// data (with `IntDictScheme` excluded) and one for stats. Both are restricted to the - /// configured allowed encodings at build time. + /// The builder produces two compressors: one for data and one for stats. + /// An explicitly built compressor is used as configured. pub fn with_btrblocks_builder(mut self, builder: BtrBlocksCompressorBuilder) -> Self { self.compressor = CompressorConfig::BtrBlocks(builder); self @@ -182,24 +165,7 @@ impl WriteStrategyBuilder { Arc::new(FlatLayoutStrategy::default()) }; - // Restrict the compressor to the allowed encodings so no compressor derived below (data, - // stats, and the defaulted probe compressor) can produce an encoding outside the policy, - // regardless of the order in which the builder and the policy were configured. - let compressor = match self.compressor { - CompressorConfig::BtrBlocks(builder) => { - CompressorConfig::BtrBlocks(match &self.allow_encodings { - Some(allow_encodings) => builder.retain_allowed_encodings(allow_encodings), - None => builder, - }) - } - opaque => opaque, - }; - - let flat: Arc = if let Some(allow_encodings) = self.allow_encodings { - Arc::new(LayoutStrategyEncodingValidator::new(flat, allow_encodings)) - } else { - flat - }; + let compressor = self.compressor; // 7. for each chunk create a flat layout let chunked = ChunkedLayoutStrategy::new(Arc::clone(&flat)); diff --git a/vortex-file/src/writer.rs b/vortex-file/src/writer.rs index e57bcb0ec86..ec45653f5c1 100644 --- a/vortex-file/src/writer.rs +++ b/vortex-file/src/writer.rs @@ -17,17 +17,20 @@ use futures::pin_mut; use futures::select; use itertools::Itertools; use vortex_array::ArrayContext; +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::dtype::DType; use vortex_array::dtype::FieldPath; use vortex_array::expr::stats::Stat; use vortex_array::iter::ArrayIterator; use vortex_array::iter::ArrayIteratorExt; +use vortex_array::session::ArraySessionExt; use vortex_array::stats::PRUNING_STATS; use vortex_array::stream::ArrayStream; use vortex_array::stream::ArrayStreamAdapter; use vortex_array::stream::ArrayStreamExt; use vortex_array::stream::SendableArrayStream; +use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_buffer::ByteBuffer; use vortex_edition::ComponentKind; use vortex_edition::EditionSessionExt; @@ -68,15 +71,16 @@ use crate::segments::writer::BufferedSegmentSink; /// Configure a new writer, which can eventually be used to write an [`ArrayStream`] into a sink /// that implements [`VortexWrite`]. /// -/// All write strategies are restricted to the components in the session's enabled editions: an -/// array, layout, extension dtype, or zone-map aggregate outside them fails the write. An empty -/// component set therefore forbids writing any component of that kind. +/// Serialized arrays, layouts, extension dtypes, and zone-map aggregates are restricted to the +/// component IDs in the session's enabled editions unless edition enforcement is explicitly +/// disabled. An empty component set therefore forbids writing any component of that kind. /// /// Construct with [`WriteOptionsSessionExt::write_options`] for normal use so the writer inherits /// the session's runtime, array registry, and memory configuration. pub struct VortexWriteOptions { session: VortexSession, - strategy: Arc, + strategy: Option>, + disable_editions: bool, buffered_bytes: BufferedBytesTracker, exclude_dtype: bool, max_variable_length_statistics_size: usize, @@ -96,16 +100,9 @@ impl WriteOptionsSessionExt for S {} impl VortexWriteOptions { /// Create a new [`VortexWriteOptions`] with the given session. pub fn new(session: VortexSession) -> Self { - let strategy = WriteStrategyBuilder::default() - .with_allow_encodings( - session - .enabled_component_ids(ComponentKind::Array) - .into_iter() - .collect(), - ) - .build(); VortexWriteOptions { - strategy, + strategy: None, + disable_editions: false, buffered_bytes: BufferedBytesTracker::new(), session, exclude_dtype: false, @@ -118,10 +115,24 @@ impl VortexWriteOptions { /// Replace the default layout strategy with the provided one. /// /// The strategy controls repartitioning, statistics layout, compression, and leaf segment - /// emission. Use [`WriteStrategyBuilder`] when only a small part of the default strategy needs - /// customization. Replacing the strategy does not change the enabled-edition encoding policy. + /// emission, and is used without being reconfigured from the enabled editions. Use + /// [`WriteStrategyBuilder`] when only a small part of the default strategy needs customization. + /// Unless edition enforcement is explicitly disabled, the final serialization context still + /// rejects array IDs not permitted by the enabled editions, independently of which in-memory + /// encodings a compressor produces. pub fn with_strategy(mut self, strategy: Arc) -> Self { - self.strategy = strategy; + self.strategy = Some(strategy); + self + } + + /// Disable all edition enforcement for this write. + /// + /// The writer permits every serialized array ID registered in the session and does not + /// restrict layout IDs, extension dtype IDs, or aggregate function IDs. This does not register + /// missing implementations, and the resulting file may not be readable by other Vortex + /// versions or configurations. + pub fn disable_editions(mut self) -> Self { + self.disable_editions = true; self } @@ -206,8 +217,9 @@ impl VortexWriteOptions { /// Note that buffers are flushed as soon as they are available with no buffering, the caller /// is responsible for deciding how to configure buffering on the underlying `Write` sink. /// - /// The set of encodings permitted in the file is snapshotted from the session's enabled - /// editions here, so editions enabled after this call do not affect the write. + /// When edition enforcement is enabled, the set of encodings permitted in the file is + /// snapshotted from the session's enabled editions here, so editions enabled after this call + /// do not affect the write. pub async fn write( self, write: W, @@ -224,13 +236,31 @@ impl VortexWriteOptions { ) -> VortexResult { validate_metadata_segments(&self.metadata)?; + let enforce_editions = !self.disable_editions; // The array context is built here, rather than when the options were constructed, so that // encodings registered on the session in between are still eligible for the file. - let ctx = LayoutWriterContext::new(new_array_context(&self.session)) - .with_buffered_bytes_tracker(self.buffered_bytes.clone()) - .with_allowed_aggregates(edition_filter(&self.session, ComponentKind::Aggregate)); + let (array_ctx, allowed_array_encodings) = + new_array_context(&self.session, enforce_editions); + let ctx = LayoutWriterContext::new(array_ctx) + .with_buffered_bytes_tracker(self.buffered_bytes.clone()); + let ctx = if enforce_editions { + ctx.with_allowed_aggregates(edition_filter(&self.session, ComponentKind::Aggregate)) + } else { + ctx + }; + let strategy = match self.strategy { + Some(strategy) => strategy, + None => WriteStrategyBuilder::default() + .with_btrblocks_builder( + BtrBlocksCompressorBuilder::default() + .retain_allowed_encodings(&allowed_array_encodings), + ) + .build(), + }; let dtype = stream.dtype().clone(); - validate_dtype_editions(&self.session, &dtype)?; + if enforce_editions { + validate_dtype_editions(&self.session, &dtype)?; + } let (mut ptr, eof) = SequenceId::root().split(); @@ -262,8 +292,7 @@ impl VortexWriteOptions { let ctx2 = ctx.clone(); let session = self.session.clone(); let layout_fut = self.session.handle().spawn_nested(move |_| async move { - let layout = self - .strategy + let layout = strategy .write_stream( ctx2, Arc::::clone(&segments), @@ -308,7 +337,7 @@ impl VortexWriteOptions { let (footer_buffers, metadata, approx_byte_size) = footer .clone() .into_serializer() - .with_layout_context(new_layout_context(&self.session)) + .with_layout_context(new_layout_context(&self.session, enforce_editions)) .with_metadata_segments(self.metadata) .with_offset(position) .with_exclude_dtype(self.exclude_dtype) @@ -355,16 +384,36 @@ impl VortexWriteOptions { } } -fn new_array_context(session: &VortexSession) -> ArrayContext { - // NOTE(os): Set up an array context with all enabled encodings pre-populated. +fn new_array_context( + session: &VortexSession, + enforce_editions: bool, +) -> (ArrayContext, HashSet) { + // NOTE(os): Set up an array context with all eligible serialized IDs pre-populated. // This is preferred for now over having an empty context here, because only the // serialised array order is deterministic. The serialisation of arrays are done // parallel and with an empty context they can register their encodings to the context // in different order, changing the written bytes from run to run. - let enabled_encoding_ids = session.enabled_component_ids(ComponentKind::Array); - ArrayContext::new(enabled_encoding_ids.iter().cloned().sorted().collect()) - // Only permit encodings in the enabled editions. - .with_allowed_ids(enabled_encoding_ids.into_iter().collect()) + let arrays = session.arrays(); + let serialized_ids = if enforce_editions { + session.enabled_component_ids(ComponentKind::Array) + } else { + arrays + .registry() + .read(|registry| registry.keys().copied().collect()) + }; + let allowed_array_encodings = serialized_ids + .iter() + .filter_map(|serialized_id| arrays.registry().get(serialized_id)) + .map(|plugin| plugin.id()) + .collect(); + let array_ctx = ArrayContext::new(serialized_ids.iter().copied().sorted().collect()); + let array_ctx = if enforce_editions { + // Only permit serialized IDs in the enabled editions. + array_ctx.with_allowed_ids(serialized_ids.into_iter().collect()) + } else { + array_ctx + }; + (array_ctx, allowed_array_encodings) } /// The ids of `kind` the enabled editions permit. @@ -419,10 +468,14 @@ fn validate_dtype_editions(session: &VortexSession, dtype: &DType) -> VortexResu validate(dtype, &allowed) } -/// The context every layout in the file is interned through, restricted to the layouts the -/// enabled editions permit. -fn new_layout_context(session: &VortexSession) -> LayoutContext { - LayoutContext::default().with_allowed_ids(edition_filter(session, ComponentKind::Layout)) +/// The context every layout in the file is interned through. +fn new_layout_context(session: &VortexSession, enforce_editions: bool) -> LayoutContext { + let context = LayoutContext::default(); + if enforce_editions { + context.with_allowed_ids(edition_filter(session, ComponentKind::Layout)) + } else { + context + } } fn validate_metadata_segments(metadata: &HashMap) -> VortexResult<()> { @@ -703,7 +756,6 @@ impl WriteSummary { #[cfg(test)] mod tests { use rstest::rstest; - use vortex_array::ArrayContext; use vortex_array::VTable; use vortex_array::array_session; use vortex_array::arrays::Bool; @@ -726,7 +778,7 @@ mod tests { static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: EDITION, - min_vortex_version: None, + min_library_version: None, }, added: &[EditionMember::array(&"vortex.primitive")], }; @@ -735,14 +787,32 @@ mod tests { session.register_edition(&DECLARATION)?; session.enable_edition(EDITION)?; - let enabled_encoding_ids = session.enabled_component_ids(ComponentKind::Array); - let ctx = ArrayContext::new(enabled_encoding_ids.clone()) - .with_allowed_ids(enabled_encoding_ids.into_iter().collect()); + let (ctx, allowed_array_encodings) = new_array_context(&session, true); assert_eq!(ctx.to_ids(), [Primitive.id()]); assert!(ctx.intern(&Bool.id()).is_none()); + assert_eq!(allowed_array_encodings, HashSet::from([Primitive.id()])); Ok(()) } + #[test] + fn disabling_editions_allows_all_registered_array_ids() { + let session = array_session(); + let (registered_ids, registered_encodings) = session.arrays().registry().read(|registry| { + ( + registry.keys().copied().sorted().collect::>(), + registry + .values() + .map(|plugin| plugin.id()) + .collect::>(), + ) + }); + + let (ctx, allowed_array_encodings) = new_array_context(&session, false); + assert_eq!(ctx.to_ids(), registered_ids); + assert_eq!(allowed_array_encodings, registered_encodings); + assert!(ctx.intern(&Bool.id()).is_some()); + } + /// This test edition declares only arrays, so every other kind must forbid all components. #[test] fn kind_filters_are_active_when_empty() -> Result<(), vortex_edition::EditionError> { @@ -750,7 +820,7 @@ mod tests { static ARRAYS_ONLY: EditionDeclaration = EditionDeclaration { edition: Edition { id: EDITION, - min_vortex_version: None, + min_library_version: None, }, added: &[EditionMember::array(&"vortex.primitive")], }; @@ -762,10 +832,15 @@ mod tests { assert!(edition_filter(&session, ComponentKind::DType).is_empty()); assert!(edition_filter(&session, ComponentKind::Aggregate).is_empty()); assert!( - new_layout_context(&session) + new_layout_context(&session, true) .intern(&"vortex.flat".into()) .is_none() ); + assert!( + new_layout_context(&session, false) + .intern(&"vortex.flat".into()) + .is_some() + ); session.editions().declare_inclusion(EditionInclusion::new( ComponentKind::Aggregate, @@ -789,7 +864,7 @@ mod tests { static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: EDITION, - min_vortex_version: None, + min_library_version: None, }, added: &[EditionMember::dtype(&"vortex.date")], }; diff --git a/vortex-file/tests/common/mod.rs b/vortex-file/tests/common/mod.rs index 4e1c85b030d..29b8ac0024f 100644 --- a/vortex-file/tests/common/mod.rs +++ b/vortex-file/tests/common/mod.rs @@ -21,7 +21,7 @@ pub fn enable_all_registered_array_encodings(session: &VortexSession) { editions .declare_edition(Edition { id: TEST_EDITION, - min_vortex_version: None, + min_library_version: None, }) .map_err(|error| vortex_err!("{error}")) .vortex_expect("test edition is valid"); diff --git a/vortex-json/src/editions.rs b/vortex-json/src/editions.rs index b3073c710fe..a6630352eb7 100644 --- a/vortex-json/src/editions.rs +++ b/vortex-json/src/editions.rs @@ -3,15 +3,24 @@ //! The `json` edition family. //! -//! JSON support is opt-in: a reader without this crate cannot resolve `vortex.json`, so the dtype -//! lives in its own family rather than in `core`. [`crate::initialize`] registers and enables the -//! edition together with the dtype plugin. +//! JSON support is opt-in. [`crate::initialize`] registers and enables the edition together with +//! the dtype plugin. use vortex_edition::Edition; use vortex_edition::EditionDeclaration; +use vortex_edition::EditionFamily; use vortex_edition::EditionId; use vortex_edition::EditionMember; +/// The `json` family: the JSON extension dtype. +pub static FAMILY: EditionFamily = EditionFamily { + name: "json", + origin: "vortex-json", + doc: "The JSON extension dtype. JSON support is opt-in: a reader built without \ +`vortex-json` cannot resolve `vortex.json`, so the dtype is versioned independently of \ +`core` and a session enables this family only by initializing the crate.", +}; + /// The August 2026 draft edition of the `json` family. pub const JSON_2026_08: EditionId = EditionId::new("json", 2026, 8, 0); @@ -23,7 +32,7 @@ pub const JSON_2026_08: EditionId = EditionId::new("json", 2026, 8, 0); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: JSON_2026_08, - min_vortex_version: None, + min_library_version: None, }, added: &[EditionMember::dtype(&"vortex.json")], }; diff --git a/vortex-json/src/lib.rs b/vortex-json/src/lib.rs index 0a117eb35e7..0cf27f2db2f 100644 --- a/vortex-json/src/lib.rs +++ b/vortex-json/src/lib.rs @@ -39,6 +39,11 @@ pub fn initialize(session: &VortexSession) { // JSON is an opt-in durable dtype, so it belongs to an independently enabled edition family. // `initialize` is idempotent, hence the guard around declaration registration. if session.editions().find(&editions::JSON_2026_08).is_none() { + session + .editions() + .declare_family(&editions::FAMILY) + .map_err(|error| vortex_err!("{error}")) + .vortex_expect("JSON edition family is valid"); session .register_edition(&editions::DECLARATION) .map_err(|error| vortex_err!("{error}")) diff --git a/vortex-layout/src/layouts/flat/writer.rs b/vortex-layout/src/layouts/flat/writer.rs index 9761c71f9ae..790e5dd6663 100644 --- a/vortex-layout/src/layouts/flat/writer.rs +++ b/vortex-layout/src/layouts/flat/writer.rs @@ -181,8 +181,6 @@ mod tests { use vortex_array::VortexSessionExecute; use vortex_array::array_session; use vortex_array::arrays::BoolArray; - use vortex_array::arrays::Dict; - use vortex_array::arrays::DictArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; use vortex_array::arrays::struct_::StructArrayExt; @@ -197,19 +195,14 @@ mod tests { use vortex_array::expr::stats::Stat; use vortex_array::expr::stats::StatsProviderExt; use vortex_array::validity::Validity; - use vortex_array::vtable::VTable; use vortex_buffer::BitBufferMut; use vortex_buffer::buffer; use vortex_error::VortexExpect; - use vortex_error::VortexResult; use vortex_io::runtime::single::block_on; use vortex_io::session::RuntimeSessionExt; use vortex_mask::AllOr; - use vortex_mask::Mask; - use vortex_utils::aliases::hash_set::HashSet; use crate::LayoutStrategy; - use crate::LayoutStrategyEncodingValidator; use crate::layouts::flat::writer::FlatLayoutStrategy; use crate::segments::TestSegments; use crate::sequence::SequenceId; @@ -411,81 +404,4 @@ mod tests { assert_eq!(field_b.as_slice::(), &[3, 4]); }) } - - #[test] - fn flat_invalid_array_fails() -> VortexResult<()> { - block_on(|handle| async { - let session = new_session().with_handle(handle); - let prim: PrimitiveArray = (0..10).collect(); - let filter = prim.filter(Mask::from_indices(10, vec![2, 3]))?; - - let ctx = ArrayContext::empty(); - - // Write the array into a byte buffer. - let (layout, _segments) = { - let segments = Arc::new(TestSegments::default()); - let (ptr, eof) = SequenceId::root().split(); - // Disallow all encodings so filter arrays fail normalization immediately. - let allowed = HashSet::default(); - let layout = - LayoutStrategyEncodingValidator::new(FlatLayoutStrategy::default(), allowed) - .write_stream( - ctx.into(), - Arc::::clone(&segments), - filter.into_array().to_array_stream().sequenced(ptr), - eof, - &session, - ) - .await; - - (layout, segments) - }; - - let err = layout.expect_err("expected error"); - assert!( - err.to_string() - .contains("normalize forbids encoding (vortex.filter)"), - "unexpected error: {err}" - ); - - Ok(()) - }) - } - - #[test] - fn flat_valid_array_writes() -> VortexResult<()> { - block_on(|handle| async { - let session = new_session().with_handle(handle); - let codes: PrimitiveArray = (0u32..10).collect(); - let values: PrimitiveArray = (0..10).collect(); - let dict = DictArray::new(codes.into_array(), values.into_array()); - - let ctx = ArrayContext::empty(); - - // Write the array into a byte buffer. - let (layout, _segments) = { - let segments = Arc::new(TestSegments::default()); - let (ptr, eof) = SequenceId::root().split(); - // Only allow the dict encoding; canonical primitive children remain permitted. - let mut allowed = HashSet::default(); - allowed.insert(Dict.id()); - let layout = - LayoutStrategyEncodingValidator::new(FlatLayoutStrategy::default(), allowed) - .write_stream( - ctx.into(), - Arc::::clone(&segments), - dict.into_array().to_array_stream().sequenced(ptr), - eof, - &session, - ) - .await; - - (layout, segments) - }; - - assert!(layout.is_ok()); - - Ok(()) - }) - } } diff --git a/vortex-layout/src/strategy.rs b/vortex-layout/src/strategy.rs index 5a0b1025e4a..4ff4b748936 100644 --- a/vortex-layout/src/strategy.rs +++ b/vortex-layout/src/strategy.rs @@ -6,12 +6,8 @@ use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; use async_trait::async_trait; -use futures::StreamExt; use vortex_array::ArrayContext; -use vortex_array::ArrayId; use vortex_array::aggregate_fn::AggregateFnId; -use vortex_array::normalize::NormalizeOptions; -use vortex_array::normalize::Operation; use vortex_error::VortexResult; use vortex_session::VortexSession; use vortex_utils::aliases::hash_set::HashSet; @@ -20,8 +16,6 @@ use crate::LayoutRef; use crate::segments::SegmentSinkRef; use crate::sequence::SendableSequentialStream; use crate::sequence::SequencePointer; -use crate::sequence::SequentialStreamAdapter; -use crate::sequence::SequentialStreamExt; /// A shared counter of the bytes that layout strategies are holding but have not yet emitted. /// @@ -199,59 +193,6 @@ pub trait LayoutStrategy: 'static + Send + Sync { ) -> VortexResult; } -/// A layout strategy wrapper that rejects arrays containing encodings outside an allow-list. -/// -/// Canonical encodings are always permitted. Every chunk is recursively validated before it is -/// passed to the wrapped strategy. -#[derive(Clone)] -pub struct LayoutStrategyEncodingValidator { - child: Arc, - allowed_encodings: Arc>, -} - -impl LayoutStrategyEncodingValidator { - /// Creates a validator around `child` using the supplied encoding allow-list. - pub fn new(child: S, allowed_encodings: HashSet) -> Self { - Self { - child: Arc::new(child), - allowed_encodings: Arc::new(allowed_encodings), - } - } -} - -#[async_trait] -impl LayoutStrategy for LayoutStrategyEncodingValidator { - async fn write_stream( - &self, - ctx: LayoutWriterContext, - segment_sink: SegmentSinkRef, - stream: SendableSequentialStream, - eof: SequencePointer, - session: &VortexSession, - ) -> VortexResult { - let dtype = stream.dtype().clone(); - let allowed_encodings = Arc::clone(&self.allowed_encodings); - let stream = stream.map(move |chunk| { - let (sequence_id, chunk) = chunk?; - let chunk = chunk.normalize(&mut NormalizeOptions { - allowed: &allowed_encodings, - operation: Operation::Error, - })?; - Ok((sequence_id, chunk)) - }); - - self.child - .write_stream( - ctx, - segment_sink, - SequentialStreamAdapter::new(dtype, stream).sendable(), - eof, - session, - ) - .await - } -} - #[async_trait] impl LayoutStrategy for Arc { async fn write_stream( diff --git a/vortex-python-cuda/src/lib.rs b/vortex-python-cuda/src/lib.rs index dff0a0fd03d..68f836be4e4 100644 --- a/vortex-python-cuda/src/lib.rs +++ b/vortex-python-cuda/src/lib.rs @@ -26,6 +26,7 @@ use pyo3::types::PyDict; use pyo3::types::PyList; use pyo3::types::PyTuple; use vortex::VortexSessionDefault; +use vortex::array::ArrayDeserialization; use vortex::array::ArrayId; use vortex::array::ArrayRef; use vortex::array::buffer::BufferHandle; @@ -300,11 +301,14 @@ fn deserialize_metadata_tree( .get(&encoding_id) .ok_or_else(|| vortex_err!("Unknown array encoding: {}", metadata.encoding_id))?; let decoded = plugin.deserialize( - &dtype, - metadata.len, - &metadata.metadata, - &metadata.buffers, - &children, + ArrayDeserialization::new( + encoding_id, + &dtype, + metadata.len, + &metadata.metadata, + &metadata.buffers, + &children, + ), session, )?; vortex_ensure!( diff --git a/vortex-python/src/arrays/mod.rs b/vortex-python/src/arrays/mod.rs index 5daf57310d3..834b42ef718 100644 --- a/vortex-python/src/arrays/mod.rs +++ b/vortex-python/src/arrays/mod.rs @@ -147,23 +147,25 @@ fn array_metadata_tuple<'py>( py: Python<'py>, array: &ArrayRef, ) -> PyVortexResult> { - let metadata = session().array_serialize(array)?.ok_or_else(|| { + let serialization = session().array_serialize(array)?.ok_or_else(|| { PyValueError::new_err(format!( - "Array {} does not support metadata serialization", + "Array {} does not support serialization", array.encoding_id() )) })?; let dtype = array.dtype().write_flatbuffer_bytes()?; - let buffers = array - .buffer_handles() + let buffers = serialization + .buffers .iter() - .map(|handle| export_buffer(py, handle).map(|cap| cap.into_any())) + .map(|buffer| { + export_buffer(py, &BufferHandle::new_host(buffer.clone())).map(|cap| cap.into_any()) + }) .collect::>>()?; let buffers = PyList::new(py, buffers)?; - let children = array - .children() + let children = serialization + .children .iter() .map(|child| array_metadata_tuple(py, child).map(|tuple| tuple.into_any())) .collect::>>()?; @@ -172,10 +174,12 @@ fn array_metadata_tuple<'py>( PyTuple::new( py, [ - array.encoding_id().to_string().into_py_any(py)?, + serialization.serialized_id.to_string().into_py_any(py)?, PyBytes::new(py, dtype.as_slice()).into_any().into(), array.len().into_py_any(py)?, - PyBytes::new(py, metadata.as_slice()).into_any().into(), + PyBytes::new(py, serialization.metadata.as_slice()) + .into_any() + .into(), buffers.into_any().into(), children.into_any().into(), ], diff --git a/vortex-python/src/io.rs b/vortex-python/src/io.rs index a9a1d2017ad..b5f2d2bcfbd 100644 --- a/vortex-python/src/io.rs +++ b/vortex-python/src/io.rs @@ -313,7 +313,7 @@ impl PyVortexWriteOptions { /// >>> vx.io.VortexWriteOptions.default().write(sprl, "chonky.vortex") /// >>> import os /// >>> os.path.getsize('chonky.vortex') - /// 215684 + /// 215788 /// /// Wow, Vortex manages to use about two bytes per integer! So advanced. So tiny. /// @@ -323,7 +323,7 @@ impl PyVortexWriteOptions { /// /// >>> vx.io.VortexWriteOptions.compact().write(sprl, "tiny.vortex") /// >>> os.path.getsize('tiny.vortex') - /// 54888 + /// 54992 /// /// Random numbers are not (usually) composed of random bytes! #[staticmethod] diff --git a/vortex-session/src/registry.rs b/vortex-session/src/registry.rs index 330ae01f07a..fb63fa03174 100644 --- a/vortex-session/src/registry.rs +++ b/vortex-session/src/registry.rs @@ -209,8 +209,10 @@ impl Interner { /// Intern an ID, returning its index. pub fn intern(&self, id: &Id) -> Option { - if let Some(allowed) = &self.allowed - && !allowed.contains(id) + if self + .allowed + .as_ref() + .is_some_and(|allowed| !allowed.contains(id)) { // ID not permitted, cannot intern. return None; diff --git a/vortex-spatial/src/editions.rs b/vortex-spatial/src/editions.rs index e7e277d4f91..f236cd3a503 100644 --- a/vortex-spatial/src/editions.rs +++ b/vortex-spatial/src/editions.rs @@ -3,16 +3,25 @@ //! The `spatial` edition family. //! -//! Spatial support is opt-in: a reader without this crate cannot resolve `vortex.st.*`, so -//! spatial members live in their own family rather than in `core`. [`crate::initialize`] -//! registers and enables the edition so the writer can serialize spatial dtypes and the AABB -//! zone stat registered by the crate. +//! Spatial support is opt-in. [`crate::initialize`] registers and enables the edition so the +//! writer can serialize spatial dtypes and the AABB zone stat registered by the crate. use vortex_edition::Edition; use vortex_edition::EditionDeclaration; +use vortex_edition::EditionFamily; use vortex_edition::EditionId; use vortex_edition::EditionMember; +/// The `spatial` family: the geometry dtypes and the AABB zone aggregate. +pub static FAMILY: EditionFamily = EditionFamily { + name: "spatial", + origin: "vortex-spatial", + doc: "The geometry extension dtypes and the axis-aligned bounding-box zone aggregate. \ +Spatial support is opt-in: a reader built without `vortex-spatial` cannot resolve \ +`vortex.st.*`, so these members are versioned independently of `core` and a session enables \ +this family only by initializing the crate.", +}; + /// The August 2026 draft edition of the `spatial` family. pub const SPATIAL_2026_08: EditionId = EditionId::new("spatial", 2026, 8, 0); @@ -23,7 +32,7 @@ pub const SPATIAL_2026_08: EditionId = EditionId::new("spatial", 2026, 8, 0); pub static DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: SPATIAL_2026_08, - min_vortex_version: None, + min_library_version: None, }, added: &[ EditionMember::dtype(&"vortex.st.box"), diff --git a/vortex-spatial/src/lib.rs b/vortex-spatial/src/lib.rs index 0d7bddf3bed..5a7523d3d97 100644 --- a/vortex-spatial/src/lib.rs +++ b/vortex-spatial/src/lib.rs @@ -101,6 +101,11 @@ pub fn initialize(session: &VortexSession) { .find(&editions::SPATIAL_2026_08) .is_none() { + session + .editions() + .declare_family(&editions::FAMILY) + .map_err(|error| vortex_err!("{error}")) + .vortex_expect("spatial edition family is valid"); session .register_edition(&editions::DECLARATION) .map_err(|error| vortex_err!("{error}")) diff --git a/vortex-tensor/Cargo.toml b/vortex-tensor/Cargo.toml index abdca676775..cf0e61ee81b 100644 --- a/vortex-tensor/Cargo.toml +++ b/vortex-tensor/Cargo.toml @@ -21,6 +21,7 @@ vortex-array = { workspace = true } vortex-arrow = { workspace = true } vortex-buffer = { workspace = true } vortex-compressor = { workspace = true } +vortex-edition = { workspace = true } vortex-error = { workspace = true } vortex-session = { workspace = true } diff --git a/vortex-tensor/src/editions.rs b/vortex-tensor/src/editions.rs new file mode 100644 index 00000000000..7979d8cdf16 --- /dev/null +++ b/vortex-tensor/src/editions.rs @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The `tensor` edition family. +//! +//! Tensor support is opt-in. This module declares its extension dtypes and persisted array +//! encodings together. + +use vortex_edition::Edition; +use vortex_edition::EditionDeclaration; +use vortex_edition::EditionFamily; +use vortex_edition::EditionId; +use vortex_edition::EditionMember; + +/// The `tensor` family: tensor extension dtypes and persisted tensor array encodings. +pub static FAMILY: EditionFamily = EditionFamily { + name: "tensor", + origin: "vortex-tensor", + doc: "Tensor extension dtypes and persisted tensor array encodings. Tensor support is \ +opt-in: a reader built without `vortex-tensor` cannot resolve these members, so they are \ +versioned independently of `core` and enabled only when the crate is initialized.", +}; + +/// The April 2026 draft edition of the `tensor` family. +pub const TENSOR_2026_04: EditionId = EditionId::new("tensor", 2026, 4, 0); + +/// The declaration of [`TENSOR_2026_04`] and the tensor components that join the family at it. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: TENSOR_2026_04, + min_library_version: None, + }, + added: &[ + EditionMember::array(&"vortex.tensor.cosine_similarity"), + EditionMember::array(&"vortex.tensor.inner_product"), + EditionMember::array(&"vortex.tensor.l2_norm"), + EditionMember::array(&"vortex.tensor.normalized"), + EditionMember::dtype(&"vortex.tensor.fixed_shape_tensor"), + EditionMember::dtype(&"vortex.tensor.vector"), + ], +}; + +#[cfg(test)] +mod tests { + use vortex_edition::EditionError; + use vortex_edition::EditionSessionExt; + use vortex_edition::test_harness::validate_edition; + + use super::*; + + #[test] + fn tensor_edition_is_valid() -> Result<(), EditionError> { + let session = vortex_array::array_session(); + crate::initialize(&session); + validate_edition(&session.editions(), &TENSOR_2026_04) + } +} diff --git a/vortex-tensor/src/encodings/normalized/tests.rs b/vortex-tensor/src/encodings/normalized/tests.rs index 0687e1ef750..41551b611d8 100644 --- a/vortex-tensor/src/encodings/normalized/tests.rs +++ b/vortex-tensor/src/encodings/normalized/tests.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use rstest::rstest; +use vortex_array::ArrayDeserialization; use vortex_array::ArrayPlugin; use vortex_array::ArrayRef; use vortex_array::ArrayVTable; @@ -652,16 +653,19 @@ fn serde_round_trip(#[case] input: ArrayRef) -> VortexResult<()> { let original = normalize(input, &mut ctx)?.into_array(); let children: Vec = original.children(); - let metadata = SESSION + let serialization = SESSION .array_serialize(&original)? .expect("Normalized must serialize"); let recovered = ArrayPlugin::deserialize( &Normalized, - original.dtype(), - original.len(), - &metadata, - &[], - &children, + ArrayDeserialization::new( + ArrayVTable::id(&Normalized), + original.dtype(), + original.len(), + &serialization.metadata, + &[], + &children, + ), &SESSION, )?; @@ -678,10 +682,13 @@ fn serialization_carries_no_metadata() -> VortexResult<()> { let non_nullable = normalize(vector_array(2, &[3.0, 4.0, 1.0, 0.0])?, &mut ctx)?.into_array(); for array in [&nullable, &non_nullable] { - let bytes = SESSION + let serialization = SESSION .array_serialize(array)? .expect("Normalized must serialize"); - assert!(bytes.is_empty(), "Normalized must not serialize metadata"); + assert!( + serialization.metadata.is_empty(), + "Normalized must not serialize metadata" + ); } assert_eq!(nullable.nchildren(), NormalizedSlots::COUNT); @@ -705,16 +712,19 @@ fn serde_round_trip_of_a_nullable_column_with_no_null_rows() -> VortexResult<()> assert!(original.dtype().is_nullable()); assert_eq!(children.len(), NormalizedSlots::COUNT - 1); - let metadata = SESSION + let serialization = SESSION .array_serialize(&original)? .expect("Normalized must serialize"); let recovered = ArrayPlugin::deserialize( &Normalized, - original.dtype(), - original.len(), - &metadata, - &[], - &children, + ArrayDeserialization::new( + ArrayVTable::id(&Normalized), + original.dtype(), + original.len(), + &serialization.metadata, + &[], + &children, + ), &SESSION, )?; @@ -734,8 +744,12 @@ fn deserialize_rejects_validity_child_for_non_nullable_dtype() -> VortexResult<( BoolArray::from_iter([true, false]).into_array(), ]; - let error = ArrayPlugin::deserialize(&Normalized, &dtype, 2, &[], &[], &children, &SESSION) - .unwrap_err(); + let error = ArrayPlugin::deserialize( + &Normalized, + ArrayDeserialization::new(ArrayVTable::id(&Normalized), &dtype, 2, &[], &[], &children), + &SESSION, + ) + .unwrap_err(); assert!( error diff --git a/vortex-tensor/src/lib.rs b/vortex-tensor/src/lib.rs index fe56827c15b..0fae75dfdd8 100644 --- a/vortex-tensor/src/lib.rs +++ b/vortex-tensor/src/lib.rs @@ -17,6 +17,9 @@ use vortex_array::dtype::session::DTypeSessionExt; use vortex_array::scalar_fn::session::ScalarFnSessionExt; use vortex_array::session::ArraySessionExt; use vortex_arrow::ArrowSessionExt; +use vortex_edition::EditionSessionExt; +use vortex_error::VortexExpect; +use vortex_error::vortex_err; use vortex_session::VortexSession; use crate::encodings::normalized::Normalized; @@ -26,6 +29,7 @@ use crate::scalar_fns::l2_norm::L2Norm; use crate::types::fixed_shape_tensor::FixedShapeTensor; use crate::types::vector::Vector; +pub mod editions; pub mod matcher; pub mod scalar_fns; @@ -78,6 +82,22 @@ pub fn initialize(session: &VortexSession) { session_arrays.register(ScalarFnArrayPlugin::new(InnerProduct)); session_arrays.register(ScalarFnArrayPlugin::new(L2Norm)); } + + if session.editions().find(&editions::TENSOR_2026_04).is_none() { + session + .editions() + .declare_family(&editions::FAMILY) + .map_err(|error| vortex_err!("{error}")) + .vortex_expect("tensor edition family is valid"); + session + .register_edition(&editions::DECLARATION) + .map_err(|error| vortex_err!("{error}")) + .vortex_expect("tensor edition declaration is valid"); + } + session + .enable_edition(editions::TENSOR_2026_04) + .map_err(|error| vortex_err!("{error}")) + .vortex_expect("tensor edition is registered"); } #[cfg(test)] diff --git a/vortex-tensor/src/scalar_fns/cosine_similarity.rs b/vortex-tensor/src/scalar_fns/cosine_similarity.rs index 9f21653b4a2..1ee4319e95f 100644 --- a/vortex-tensor/src/scalar_fns/cosine_similarity.rs +++ b/vortex-tensor/src/scalar_fns/cosine_similarity.rs @@ -310,6 +310,7 @@ impl CosineSimilarity { mod tests { use rstest::rstest; + use vortex_array::ArrayDeserialization; use vortex_array::ArrayPlugin; use vortex_array::ArrayRef; use vortex_array::IntoArray; @@ -755,17 +756,20 @@ mod tests { let original = CosineSimilarity::try_new(lhs.clone(), rhs.clone())?.into_array(); let plugin = ScalarFnArrayPlugin::new(CosineSimilarity); - let metadata = plugin + let serialization = plugin .serialize(&original, &SESSION)? .expect("CosineSimilarity serialize must produce metadata"); let children = vec![lhs, rhs]; let recovered = plugin.deserialize( - original.dtype(), - original.len(), - &metadata, - &[], - &children, + ArrayDeserialization::new( + plugin.id(), + original.dtype(), + original.len(), + &serialization.metadata, + &[], + &children, + ), &SESSION, )?; diff --git a/vortex-tensor/src/scalar_fns/inner_product.rs b/vortex-tensor/src/scalar_fns/inner_product.rs index f10e697732d..b5e02500042 100644 --- a/vortex-tensor/src/scalar_fns/inner_product.rs +++ b/vortex-tensor/src/scalar_fns/inner_product.rs @@ -280,6 +280,7 @@ fn inner_product_row(a: &[T], b: &[T]) -> T { mod tests { use rstest::rstest; + use vortex_array::ArrayDeserialization; use vortex_array::ArrayPlugin; use vortex_array::ArrayRef; use vortex_array::IntoArray; @@ -486,17 +487,20 @@ mod tests { let original = InnerProduct::try_new(lhs.clone(), rhs.clone())?.into_array(); let plugin = ScalarFnArrayPlugin::new(InnerProduct); - let metadata = plugin + let serialization = plugin .serialize(&original, &SESSION)? .expect("InnerProduct serialize must produce metadata"); let children = vec![lhs, rhs]; let recovered = plugin.deserialize( - original.dtype(), - original.len(), - &metadata, - &[], - &children, + ArrayDeserialization::new( + plugin.id(), + original.dtype(), + original.len(), + &serialization.metadata, + &[], + &children, + ), &SESSION, )?; diff --git a/vortex-tensor/src/scalar_fns/l2_norm.rs b/vortex-tensor/src/scalar_fns/l2_norm.rs index 0b00ec95aa4..0cff689dfdf 100644 --- a/vortex-tensor/src/scalar_fns/l2_norm.rs +++ b/vortex-tensor/src/scalar_fns/l2_norm.rs @@ -252,6 +252,7 @@ fn l2_norm_row(v: &[T]) -> T { mod tests { use rstest::rstest; + use vortex_array::ArrayDeserialization; use vortex_array::ArrayPlugin; use vortex_array::ArrayRef; use vortex_array::EmptyMetadata; @@ -430,17 +431,20 @@ mod tests { let original = L2Norm::try_new(child.clone())?.into_array(); let plugin = ScalarFnArrayPlugin::new(L2Norm); - let metadata = plugin + let serialization = plugin .serialize(&original, &SESSION)? .expect("L2Norm serialize must produce metadata"); let children = vec![child]; let recovered = plugin.deserialize( - original.dtype(), - original.len(), - &metadata, - &[], - &children, + ArrayDeserialization::new( + plugin.id(), + original.dtype(), + original.len(), + &serialization.metadata, + &[], + &children, + ), &SESSION, )?; diff --git a/vortex-test/compat-gen/src/adapter.rs b/vortex-test/compat-gen/src/adapter.rs index 89cc43c678c..9a70aea3158 100644 --- a/vortex-test/compat-gen/src/adapter.rs +++ b/vortex-test/compat-gen/src/adapter.rs @@ -61,7 +61,8 @@ pub fn write_file_to_bytes(chunk: ArrayRef) -> VortexResult { write_compressed_to_bytes(chunk, Arc::new(FlatLayoutStrategy::default())) } -/// Write a `.vortex` file using a caller-provided layout strategy (compressor pipeline). +/// Write a `.vortex` file using a caller-provided layout strategy (compressor pipeline), allowing +/// the strategy to emit uneditioned encodings. pub fn write_compressed( path: &Path, chunk: ArrayRef, @@ -76,6 +77,7 @@ pub fn write_compressed( .map_err(|e| vortex_err!("failed to create {}: {e}", path.display()))?; let _summary = session .write_options() + .disable_editions() .with_strategy(strategy) .write(&mut file, stream) .await?; @@ -91,7 +93,8 @@ pub fn write_compressed_to_bytes( write_compressed_to_bytes_with_session(&VortexSession::default(), chunk, strategy) } -/// Write a `.vortex` file into memory using a caller-provided session and layout strategy. +/// Write a `.vortex` file into memory using a caller-provided session and layout strategy, +/// allowing the strategy to emit uneditioned encodings. pub fn write_compressed_to_bytes_with_session( session: &VortexSession, chunk: ArrayRef, @@ -105,6 +108,7 @@ pub fn write_compressed_to_bytes_with_session( let mut bytes = Vec::new(); let _summary = session .write_options() + .disable_editions() .with_strategy(strategy) .write(&mut bytes, stream) .await?; diff --git a/vortex-web/crate/src/wasm.rs b/vortex-web/crate/src/wasm.rs index b22440d88b0..8aa5a1cff0c 100644 --- a/vortex-web/crate/src/wasm.rs +++ b/vortex-web/crate/src/wasm.rs @@ -650,7 +650,7 @@ fn build_array_encoding_tree_from_array( .array_serialize(array) .ok() .flatten() - .map(|m| m.len()) + .map(|serialization| serialization.metadata.len()) .unwrap_or(0); let named_children = array.named_children(); diff --git a/vortex/Cargo.toml b/vortex/Cargo.toml index fc3f9e420f8..e90f6f6e8e7 100644 --- a/vortex/Cargo.toml +++ b/vortex/Cargo.toml @@ -43,6 +43,7 @@ vortex-ipc = { workspace = true } vortex-layout = { workspace = true } vortex-mask = { workspace = true } vortex-metrics = { workspace = true } +vortex-parquet-variant = { workspace = true } vortex-pco = { workspace = true } vortex-proto = { workspace = true, default-features = true } vortex-runend = { workspace = true } diff --git a/vortex/editions/core/core2025.05.0.toml b/vortex/editions/core/core2025.05.0.toml new file mode 100644 index 00000000000..c6407519b0e --- /dev/null +++ b/vortex/editions/core/core2025.05.0.toml @@ -0,0 +1,93 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting a frozen one is rejected by CI. + +edition = "core2025.05.0" +family = "core" +origin = "vortex" +min_library_version = "0.36.0" + +# Components added by this edition. +[added] +arrays = [ + "fastlanes.bitpacked", + "fastlanes.for", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fsst", + "vortex.list", + "vortex.null", + "vortex.primitive", + "vortex.runend", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", +] +layouts = [ + "vortex.chunked", + "vortex.dict", + "vortex.flat", + "vortex.stats", + "vortex.struct", +] +dtypes = [ + "vortex.date", + "vortex.time", + "vortex.timestamp", +] +aggregates = [] + +# The edition's full membership: the members above, plus every member of earlier +# editions of the family. +[components] +arrays = [ + "fastlanes.bitpacked", + "fastlanes.for", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fsst", + "vortex.list", + "vortex.null", + "vortex.primitive", + "vortex.runend", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", +] +layouts = [ + "vortex.chunked", + "vortex.dict", + "vortex.flat", + "vortex.stats", + "vortex.struct", +] +dtypes = [ + "vortex.date", + "vortex.time", + "vortex.timestamp", +] +aggregates = [] diff --git a/vortex/editions/core/core2025.06.0.toml b/vortex/editions/core/core2025.06.0.toml new file mode 100644 index 00000000000..a8c595275f6 --- /dev/null +++ b/vortex/editions/core/core2025.06.0.toml @@ -0,0 +1,66 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting a frozen one is rejected by CI. + +edition = "core2025.06.0" +family = "core" +origin = "vortex" +min_library_version = "0.40.0" + +# Components added by this edition. +[added] +arrays = [ + "vortex.pco", + "vortex.sequence", + "vortex.zstd", +] +layouts = [] +dtypes = [] +aggregates = [] + +# The edition's full membership: the members above, plus every member of earlier +# editions of the family. +[components] +arrays = [ + "fastlanes.bitpacked", + "fastlanes.for", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fsst", + "vortex.list", + "vortex.null", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", + "vortex.zstd", +] +layouts = [ + "vortex.chunked", + "vortex.dict", + "vortex.flat", + "vortex.stats", + "vortex.struct", +] +dtypes = [ + "vortex.date", + "vortex.time", + "vortex.timestamp", +] +aggregates = [] diff --git a/vortex/editions/core/core2025.10.0.toml b/vortex/editions/core/core2025.10.0.toml new file mode 100644 index 00000000000..22e9715e387 --- /dev/null +++ b/vortex/editions/core/core2025.10.0.toml @@ -0,0 +1,71 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting a frozen one is rejected by CI. + +edition = "core2025.10.0" +family = "core" +origin = "vortex" +min_library_version = "0.54.0" + +# Components added by this edition. +[added] +arrays = [ + "fastlanes.rle", + "vortex.fixed_size_list", + "vortex.listview", + "vortex.masked", +] +layouts = [] +dtypes = [] +aggregates = [] + +# The edition's full membership: the members above, plus every member of earlier +# editions of the family. +[components] +arrays = [ + "fastlanes.bitpacked", + "fastlanes.for", + "fastlanes.rle", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fixed_size_list", + "vortex.fsst", + "vortex.list", + "vortex.listview", + "vortex.masked", + "vortex.null", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", + "vortex.zstd", +] +layouts = [ + "vortex.chunked", + "vortex.dict", + "vortex.flat", + "vortex.stats", + "vortex.struct", +] +dtypes = [ + "vortex.date", + "vortex.time", + "vortex.timestamp", +] +aggregates = [] diff --git a/vortex/editions/core/core2026.08.0.toml b/vortex/editions/core/core2026.08.0.toml new file mode 100644 index 00000000000..24d93b9ec27 --- /dev/null +++ b/vortex/editions/core/core2026.08.0.toml @@ -0,0 +1,83 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting a frozen one is rejected by CI. + +edition = "core2026.08.0" +family = "core" +origin = "vortex" +min_library_version = "0.84.0" + +# Components added by this edition. +[added] +arrays = [] +layouts = [ + "vortex.zoned", +] +dtypes = [] +aggregates = [ + "vortex.bounded_max", + "vortex.bounded_min", + "vortex.max", + "vortex.min", + "vortex.nan_count", + "vortex.null_count", +] + +# The edition's full membership: the members above, plus every member of earlier +# editions of the family. +[components] +arrays = [ + "fastlanes.bitpacked", + "fastlanes.for", + "fastlanes.rle", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fixed_size_list", + "vortex.fsst", + "vortex.list", + "vortex.listview", + "vortex.masked", + "vortex.null", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", + "vortex.zstd", +] +layouts = [ + "vortex.chunked", + "vortex.dict", + "vortex.flat", + "vortex.stats", + "vortex.struct", + "vortex.zoned", +] +dtypes = [ + "vortex.date", + "vortex.time", + "vortex.timestamp", +] +aggregates = [ + "vortex.bounded_max", + "vortex.bounded_min", + "vortex.max", + "vortex.min", + "vortex.nan_count", + "vortex.null_count", +] diff --git a/vortex/editions/core/core2026.08.1.toml b/vortex/editions/core/core2026.08.1.toml new file mode 100644 index 00000000000..814074a25f4 --- /dev/null +++ b/vortex/editions/core/core2026.08.1.toml @@ -0,0 +1,77 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting a frozen one is rejected by CI. + +edition = "core2026.08.1" +family = "core" +origin = "vortex" +min_library_version = "0.84.0" + +# Components added by this edition. +[added] +arrays = [ + "vortex.onpair", +] +layouts = [] +dtypes = [] +aggregates = [] + +# The edition's full membership: the members above, plus every member of earlier +# editions of the family. +[components] +arrays = [ + "fastlanes.bitpacked", + "fastlanes.for", + "fastlanes.rle", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fixed_size_list", + "vortex.fsst", + "vortex.list", + "vortex.listview", + "vortex.masked", + "vortex.null", + "vortex.onpair", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", + "vortex.zstd", +] +layouts = [ + "vortex.chunked", + "vortex.dict", + "vortex.flat", + "vortex.stats", + "vortex.struct", + "vortex.zoned", +] +dtypes = [ + "vortex.date", + "vortex.time", + "vortex.timestamp", +] +aggregates = [ + "vortex.bounded_max", + "vortex.bounded_min", + "vortex.max", + "vortex.min", + "vortex.nan_count", + "vortex.null_count", +] diff --git a/vortex/editions/core/core2026.08.2.toml b/vortex/editions/core/core2026.08.2.toml new file mode 100644 index 00000000000..5a21a0a4e73 --- /dev/null +++ b/vortex/editions/core/core2026.08.2.toml @@ -0,0 +1,78 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting a frozen one is rejected by CI. + +edition = "core2026.08.2" +family = "core" +origin = "vortex" +min_library_version = "0.85.0" + +# Components added by this edition. +[added] +arrays = [ + "vortex.map", +] +layouts = [] +dtypes = [] +aggregates = [] + +# The edition's full membership: the members above, plus every member of earlier +# editions of the family. +[components] +arrays = [ + "fastlanes.bitpacked", + "fastlanes.for", + "fastlanes.rle", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fixed_size_list", + "vortex.fsst", + "vortex.list", + "vortex.listview", + "vortex.map", + "vortex.masked", + "vortex.null", + "vortex.onpair", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.zigzag", + "vortex.zstd", +] +layouts = [ + "vortex.chunked", + "vortex.dict", + "vortex.flat", + "vortex.stats", + "vortex.struct", + "vortex.zoned", +] +dtypes = [ + "vortex.date", + "vortex.time", + "vortex.timestamp", +] +aggregates = [ + "vortex.bounded_max", + "vortex.bounded_min", + "vortex.max", + "vortex.min", + "vortex.nan_count", + "vortex.null_count", +] diff --git a/vortex/editions/core/core2026.08.3.toml b/vortex/editions/core/core2026.08.3.toml new file mode 100644 index 00000000000..284f68febba --- /dev/null +++ b/vortex/editions/core/core2026.08.3.toml @@ -0,0 +1,84 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting a frozen one is rejected by CI. + +edition = "core2026.08.3" +family = "core" +origin = "vortex" +min_library_version = "0.85.0" + +# Components added by this edition. +[added] +arrays = [ + "vortex.parquet.variant", + "vortex.variant", +] +layouts = [] +dtypes = [ + "vortex.uuid", +] +aggregates = [] + +# The edition's full membership: the members above, plus every member of earlier +# editions of the family. +[components] +arrays = [ + "fastlanes.bitpacked", + "fastlanes.for", + "fastlanes.rle", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fixed_size_list", + "vortex.fsst", + "vortex.list", + "vortex.listview", + "vortex.map", + "vortex.masked", + "vortex.null", + "vortex.onpair", + "vortex.parquet.variant", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.variant", + "vortex.zigzag", + "vortex.zstd", +] +layouts = [ + "vortex.chunked", + "vortex.dict", + "vortex.flat", + "vortex.stats", + "vortex.struct", + "vortex.zoned", +] +dtypes = [ + "vortex.date", + "vortex.time", + "vortex.timestamp", + "vortex.uuid", +] +aggregates = [ + "vortex.bounded_max", + "vortex.bounded_min", + "vortex.max", + "vortex.min", + "vortex.nan_count", + "vortex.null_count", +] diff --git a/vortex/editions/core/family.toml b/vortex/editions/core/family.toml new file mode 100644 index 00000000000..154d15bc994 --- /dev/null +++ b/vortex/editions/core/family.toml @@ -0,0 +1,17 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This describes the family the editions beside it belong to. + +name = "core" +origin = "vortex" + +doc = """ +The serialized components available to the default file writer. Each array ID names a wire +representation that old readers either recognize or reject; several IDs may deserialize into +one current in-memory array. Every core edition freezes, and a frozen edition carries a +read-forever guarantee: a file written with it stays readable by every later Vortex release. +New core objects first undergo testing in independently versioned families, then join +preview for broad opt-in use, and finally join core with the same IDs and wire contracts. An +edition may freeze in the release that cuts it; after that release version is known, the +declaration is backfilled with it as the minimum. A frozen edition never changes. +""" diff --git a/vortex/editions/json/family.toml b/vortex/editions/json/family.toml new file mode 100644 index 00000000000..7c0d8569861 --- /dev/null +++ b/vortex/editions/json/family.toml @@ -0,0 +1,12 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This describes the family the editions beside it belong to. + +name = "json" +origin = "vortex-json" + +doc = """ +The JSON extension dtype. JSON support is opt-in: a reader built without `vortex-json` +cannot resolve `vortex.json`, so the dtype is versioned independently of `core` and a +session enables this family only by initializing the crate. +""" diff --git a/vortex/editions/json/json2026.08.0.toml b/vortex/editions/json/json2026.08.0.toml new file mode 100644 index 00000000000..a2131940dc5 --- /dev/null +++ b/vortex/editions/json/json2026.08.0.toml @@ -0,0 +1,29 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition record has no read-forever guarantee. It may describe an evolving feature, +# a stable component awaiting adoption, or a release waiting to be cut. New capabilities advance +# to a new edition; after an origin release is known, min_library_version is backfilled to document +# its freeze. A frozen record never changes. + +edition = "json2026.08.0" +family = "json" +origin = "vortex-json" + +# Components added by this edition. +[added] +arrays = [] +layouts = [] +dtypes = [ + "vortex.json", +] +aggregates = [] + +# The edition's full membership: the members above, plus every member of earlier +# editions of the family. +[components] +arrays = [] +layouts = [] +dtypes = [ + "vortex.json", +] +aggregates = [] diff --git a/vortex/editions/preview/family.toml b/vortex/editions/preview/family.toml new file mode 100644 index 00000000000..82c6a6a0239 --- /dev/null +++ b/vortex/editions/preview/family.toml @@ -0,0 +1,12 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This describes the family the editions beside it belong to. + +name = "preview" +origin = "vortex" + +doc = """ +Additive, opt-in components maintained as part of Vortex but not yet adopted by the default +core writer. Components enter preview only once their serialized contracts are ready for +broad testing; independently evolving work remains in its own family until then. +""" diff --git a/vortex/editions/preview/preview2026.08.0.toml b/vortex/editions/preview/preview2026.08.0.toml new file mode 100644 index 00000000000..550d1df53aa --- /dev/null +++ b/vortex/editions/preview/preview2026.08.0.toml @@ -0,0 +1,25 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition record has no read-forever guarantee. It may describe an evolving feature, +# a stable component awaiting adoption, or a release waiting to be cut. New capabilities advance +# to a new edition; after an origin release is known, min_library_version is backfilled to document +# its freeze. A frozen record never changes. + +edition = "preview2026.08.0" +family = "preview" +origin = "vortex" + +# Components added by this edition. +[added] +arrays = [] +layouts = [] +dtypes = [] +aggregates = [] + +# The edition's full membership: the members above, plus every member of earlier +# editions of the family. +[components] +arrays = [] +layouts = [] +dtypes = [] +aggregates = [] diff --git a/vortex/editions/spatial/family.toml b/vortex/editions/spatial/family.toml new file mode 100644 index 00000000000..3c4d83637bc --- /dev/null +++ b/vortex/editions/spatial/family.toml @@ -0,0 +1,13 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This describes the family the editions beside it belong to. + +name = "spatial" +origin = "vortex-spatial" + +doc = """ +The geometry extension dtypes and the axis-aligned bounding-box zone aggregate. Spatial +support is opt-in: a reader built without `vortex-spatial` cannot resolve `vortex.st.*`, so +these members are versioned independently of `core` and a session enables this family only +by initializing the crate. +""" diff --git a/vortex/editions/spatial/spatial2026.08.0.toml b/vortex/editions/spatial/spatial2026.08.0.toml new file mode 100644 index 00000000000..0f1154c83a8 --- /dev/null +++ b/vortex/editions/spatial/spatial2026.08.0.toml @@ -0,0 +1,47 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition record has no read-forever guarantee. It may describe an evolving feature, +# a stable component awaiting adoption, or a release waiting to be cut. New capabilities advance +# to a new edition; after an origin release is known, min_library_version is backfilled to document +# its freeze. A frozen record never changes. + +edition = "spatial2026.08.0" +family = "spatial" +origin = "vortex-spatial" + +# Components added by this edition. +[added] +arrays = [] +layouts = [] +dtypes = [ + "vortex.st.box", + "vortex.st.linestring", + "vortex.st.multilinestring", + "vortex.st.multipoint", + "vortex.st.multipolygon", + "vortex.st.point", + "vortex.st.polygon", + "vortex.st.wkb", +] +aggregates = [ + "vortex.st.aabb", +] + +# The edition's full membership: the members above, plus every member of earlier +# editions of the family. +[components] +arrays = [] +layouts = [] +dtypes = [ + "vortex.st.box", + "vortex.st.linestring", + "vortex.st.multilinestring", + "vortex.st.multipoint", + "vortex.st.multipolygon", + "vortex.st.point", + "vortex.st.polygon", + "vortex.st.wkb", +] +aggregates = [ + "vortex.st.aabb", +] diff --git a/vortex/editions/tensor/family.toml b/vortex/editions/tensor/family.toml new file mode 100644 index 00000000000..251545a3a28 --- /dev/null +++ b/vortex/editions/tensor/family.toml @@ -0,0 +1,12 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This describes the family the editions beside it belong to. + +name = "tensor" +origin = "vortex-tensor" + +doc = """ +Tensor extension dtypes and persisted tensor array encodings. Tensor support is opt-in: a +reader built without `vortex-tensor` cannot resolve these members, so they are versioned +independently of `core` and enabled only when the crate is initialized. +""" diff --git a/vortex/editions/tensor/tensor2026.04.0.toml b/vortex/editions/tensor/tensor2026.04.0.toml new file mode 100644 index 00000000000..2e3c226f0ba --- /dev/null +++ b/vortex/editions/tensor/tensor2026.04.0.toml @@ -0,0 +1,41 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition record has no read-forever guarantee. It may describe an evolving feature, +# a stable component awaiting adoption, or a release waiting to be cut. New capabilities advance +# to a new edition; after an origin release is known, min_library_version is backfilled to document +# its freeze. A frozen record never changes. + +edition = "tensor2026.04.0" +family = "tensor" +origin = "vortex-tensor" + +# Components added by this edition. +[added] +arrays = [ + "vortex.tensor.cosine_similarity", + "vortex.tensor.inner_product", + "vortex.tensor.l2_norm", + "vortex.tensor.normalized", +] +layouts = [] +dtypes = [ + "vortex.tensor.fixed_shape_tensor", + "vortex.tensor.vector", +] +aggregates = [] + +# The edition's full membership: the members above, plus every member of earlier +# editions of the family. +[components] +arrays = [ + "vortex.tensor.cosine_similarity", + "vortex.tensor.inner_product", + "vortex.tensor.l2_norm", + "vortex.tensor.normalized", +] +layouts = [] +dtypes = [ + "vortex.tensor.fixed_shape_tensor", + "vortex.tensor.vector", +] +aggregates = [] diff --git a/vortex/editions/zstd/family.toml b/vortex/editions/zstd/family.toml new file mode 100644 index 00000000000..408d8727aaa --- /dev/null +++ b/vortex/editions/zstd/family.toml @@ -0,0 +1,12 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This describes the family the editions beside it belong to. + +name = "zstd" +origin = "vortex-zstd" + +doc = """ +Optional Zstd-backed serialized array representations. A reader built without `vortex-zstd` +cannot resolve these members, so they are versioned independently of `core` and enabled only +when the crate is initialized with the corresponding feature. +""" diff --git a/vortex/editions/zstd/zstd2026.02.0.toml b/vortex/editions/zstd/zstd2026.02.0.toml new file mode 100644 index 00000000000..6109413c2fb --- /dev/null +++ b/vortex/editions/zstd/zstd2026.02.0.toml @@ -0,0 +1,29 @@ +# Generated by `cargo run -p xtask -- generate-editions`. +# +# This edition record has no read-forever guarantee. It may describe an evolving feature, +# a stable component awaiting adoption, or a release waiting to be cut. New capabilities advance +# to a new edition; after an origin release is known, min_library_version is backfilled to document +# its freeze. A frozen record never changes. + +edition = "zstd2026.02.0" +family = "zstd" +origin = "vortex-zstd" + +# Components added by this edition. +[added] +arrays = [ + "vortex.zstd_buffers", +] +layouts = [] +dtypes = [] +aggregates = [] + +# The edition's full membership: the members above, plus every member of earlier +# editions of the family. +[components] +arrays = [ + "vortex.zstd_buffers", +] +layouts = [] +dtypes = [] +aggregates = [] diff --git a/vortex/src/editions/core/mod.rs b/vortex/src/editions/core/mod.rs deleted file mode 100644 index 84a0dcb8a05..00000000000 --- a/vortex/src/editions/core/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! The `core` edition family: the serialized components the default file writer emits. -//! -//! One module per edition, each declaring the edition and the components that join the -//! family at it; members of earlier editions are inherited and never restated. - -pub mod v2025_05; -pub mod v2025_06; -pub mod v2025_10; -pub mod v2026_08; -pub mod v2026_08_2; -pub mod v2026_08_3; - -pub use v2025_05::CORE_2025_05_0; -pub use v2025_06::CORE_2025_06_0; -pub use v2025_10::CORE_2025_10_0; -pub use v2026_08::CORE_2026_08_0; -pub use v2026_08::CORE_2026_08_1; -pub use v2026_08_2::CORE_2026_08_2; -pub use v2026_08_3::CORE_2026_08_3; diff --git a/vortex/src/editions/mod.rs b/vortex/src/editions/mod.rs index a6dd8ee7fe9..320cb071a16 100644 --- a/vortex/src/editions/mod.rs +++ b/vortex/src/editions/mod.rs @@ -3,74 +3,68 @@ //! The Vortex edition declarations. //! -//! [`vortex_edition`] provides the types, session variables, and test harness. The actual -//! first-party declarations live here, one module per edition. The default session first -//! registers them with [`crate::editions::register_default_editions`] and then selects its write -//! policy with [`crate::editions::enable_default_editions`]. +//! [`vortex_edition`] provides the types, session variables, test harness, and the +//! first-party declarations themselves. This module re-exports them and owns the session +//! wiring: the default session first registers them with +//! [`crate::editions::register_default_editions`] and then selects its write policy with +//! [`crate::editions::enable_default_editions`]. //! -//! Members carry a [`crate::editions::ComponentKind`]: arrays a written array may use, extension -//! dtypes its schema may contain, and the aggregates zone maps record. Every kind is restricted to -//! its declared members, so an empty set permits no components of that kind. +//! Members carry a [`crate::editions::ComponentKind`]: serialized array IDs a writer may emit, +//! extension dtypes its schema may contain, and aggregates zone maps record. Array serializers +//! choose a wire representation independently of the enabled editions; the serialization context +//! rejects an ID that is not permitted unless the writer explicitly disables edition enforcement. //! //! The default file writer resolves the session's enabled editions at write time. The -//! facade enables the newest frozen `core` edition, [`crate::editions::CORE_2026_08_1`], and -//! additionally enables the latest preview edition when the `unstable_encodings` feature is -//! selected. +//! facade enables [`crate::editions::CORE_2026_08_3`] and +//! additionally enables the `preview` edition when the +//! `unstable_encodings` feature is selected. -pub mod core; -pub mod preview; #[cfg(test)] mod tests; pub use vortex_edition::ComponentKind; +pub use vortex_edition::EDITION_DECLARATIONS; +pub use vortex_edition::EDITION_FAMILIES; pub use vortex_edition::Edition; pub use vortex_edition::EditionDeclaration; +pub use vortex_edition::EditionFamily; pub use vortex_edition::EditionId; pub use vortex_edition::EditionInclusion; pub use vortex_edition::EditionMember; pub use vortex_edition::EditionSession; pub use vortex_edition::EditionSessionExt; pub use vortex_edition::EnabledEditions; +pub use vortex_edition::declarations::core; +pub use vortex_edition::declarations::core::CORE_2025_05_0; +pub use vortex_edition::declarations::core::CORE_2025_06_0; +pub use vortex_edition::declarations::core::CORE_2025_10_0; +pub use vortex_edition::declarations::core::CORE_2026_08_0; +pub use vortex_edition::declarations::core::CORE_2026_08_1; +pub use vortex_edition::declarations::core::CORE_2026_08_2; +pub use vortex_edition::declarations::core::CORE_2026_08_3; +pub use vortex_edition::declarations::preview; +pub use vortex_edition::declarations::preview::PREVIEW_2026_08_0; use vortex_error::VortexExpect; use vortex_error::vortex_err; use vortex_session::VortexSession; -pub use self::core::CORE_2025_05_0; -pub use self::core::CORE_2025_06_0; -pub use self::core::CORE_2025_10_0; -pub use self::core::CORE_2026_08_0; -pub use self::core::CORE_2026_08_1; -pub use self::core::CORE_2026_08_2; -pub use self::core::CORE_2026_08_3; -pub use self::preview::PREVIEW_2025_05_0; -pub use self::preview::PREVIEW_2026_02_0; -pub use self::preview::PREVIEW_2026_04_0; -pub use self::preview::PREVIEW_2026_06_0; - /// The `core` edition enabled for writing by the default Vortex session. -pub const DEFAULT_CORE_EDITION: EditionId = CORE_2026_08_1; +pub const DEFAULT_CORE_EDITION: EditionId = CORE_2026_08_3; /// The `preview` edition enabled for writing by the default Vortex session when the /// `unstable_encodings` feature is selected. -pub const DEFAULT_PREVIEW_EDITION: EditionId = PREVIEW_2026_06_0; - -/// The first-party Vortex edition declarations. -pub static EDITION_DECLARATIONS: &[&EditionDeclaration] = &[ - &core::v2025_05::DECLARATION, - &core::v2025_06::DECLARATION, - &core::v2025_10::DECLARATION, - &core::v2026_08::DECLARATION_0, - &core::v2026_08::DECLARATION_1, - &core::v2026_08_2::DECLARATION, - &core::v2026_08_3::DECLARATION, - &preview::v2025_05::DECLARATION, - &preview::v2026_02::DECLARATION, - &preview::v2026_04::DECLARATION, - &preview::v2026_06::DECLARATION, -]; +pub const DEFAULT_PREVIEW_EDITION: EditionId = PREVIEW_2026_08_0; -/// Register the Vortex edition declarations with the session's [`EditionSession`]. +/// Register the Vortex edition families and declarations with the session's +/// [`EditionSession`]. pub fn register_default_editions(session: &VortexSession) { + for family in EDITION_FAMILIES { + session + .editions() + .declare_family(family) + .map_err(|e| vortex_err!("{e}")) + .vortex_expect("edition families are valid"); + } for declaration in EDITION_DECLARATIONS { session .register_edition(declaration) @@ -81,8 +75,8 @@ pub fn register_default_editions(session: &VortexSession) { /// Enable the default Vortex editions for writing. /// -/// This selects the newest frozen `core` edition and, when configured, the newest preview -/// edition. All declarations must have been registered first with +/// This selects the default `core` edition and, when configured, the `preview` edition. All +/// declarations must have been registered first with /// [`register_default_editions`]. pub fn enable_default_editions(session: &VortexSession) { session @@ -94,5 +88,5 @@ pub fn enable_default_editions(session: &VortexSession) { session .enable_edition(DEFAULT_PREVIEW_EDITION) .map_err(|e| vortex_err!("{e}")) - .vortex_expect("default preview edition is registered"); + .vortex_expect("feature edition is registered"); } diff --git a/vortex/src/editions/preview/mod.rs b/vortex/src/editions/preview/mod.rs deleted file mode 100644 index cba3ba1dc6c..00000000000 --- a/vortex/src/editions/preview/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! The `preview` edition family: opt-in components without a frozen compatibility guarantee. -//! -//! One module per draft edition, each declaring the components that join the family at it. -//! Members of earlier editions are inherited and never restated. - -pub mod v2025_05; -pub mod v2026_02; -pub mod v2026_04; -pub mod v2026_06; - -pub use v2025_05::PREVIEW_2025_05_0; -pub use v2026_02::PREVIEW_2026_02_0; -pub use v2026_04::PREVIEW_2026_04_0; -pub use v2026_06::PREVIEW_2026_06_0; diff --git a/vortex/src/editions/preview/v2025_05.rs b/vortex/src/editions/preview/v2025_05.rs deleted file mode 100644 index 818ec96b15a..00000000000 --- a/vortex/src/editions/preview/v2025_05.rs +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! The May 2025 `preview` encoding cohort. - -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; -use vortex_edition::EditionMember; - -/// The May 2025 draft edition of the `preview` family. -pub const PREVIEW_2025_05_0: EditionId = EditionId::new("preview", 2025, 5, 0); - -/// The declaration of [`PREVIEW_2025_05_0`] and the encodings that join the family at it. -pub static DECLARATION: EditionDeclaration = EditionDeclaration { - edition: Edition { - id: PREVIEW_2025_05_0, - min_vortex_version: None, - }, - added: &[EditionMember::array(&"fastlanes.delta")], -}; diff --git a/vortex/src/editions/preview/v2026_02.rs b/vortex/src/editions/preview/v2026_02.rs deleted file mode 100644 index 691e96c1850..00000000000 --- a/vortex/src/editions/preview/v2026_02.rs +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! The February 2026 `preview` encoding cohort. - -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; -use vortex_edition::EditionMember; - -/// The February 2026 draft edition of the `preview` family. -pub const PREVIEW_2026_02_0: EditionId = EditionId::new("preview", 2026, 2, 0); - -/// The declaration of [`PREVIEW_2026_02_0`] and the encodings that join the family at it. -pub static DECLARATION: EditionDeclaration = EditionDeclaration { - edition: Edition { - id: PREVIEW_2026_02_0, - min_vortex_version: None, - }, - added: &[EditionMember::array(&"vortex.zstd_buffers")], -}; diff --git a/vortex/src/editions/preview/v2026_04.rs b/vortex/src/editions/preview/v2026_04.rs deleted file mode 100644 index 4d1a5ede4b8..00000000000 --- a/vortex/src/editions/preview/v2026_04.rs +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! The April 2026 `preview` component cohort. - -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; -use vortex_edition::EditionMember; - -/// The April 2026 draft edition of the `preview` family. -pub const PREVIEW_2026_04_0: EditionId = EditionId::new("preview", 2026, 4, 0); - -/// The declaration of [`PREVIEW_2026_04_0`] and the components that join the family at it. -pub static DECLARATION: EditionDeclaration = EditionDeclaration { - edition: Edition { - id: PREVIEW_2026_04_0, - min_vortex_version: None, - }, - added: &[ - EditionMember::array(&"vortex.patched"), - EditionMember::array(&"vortex.tensor.cosine_similarity"), - EditionMember::array(&"vortex.tensor.inner_product"), - EditionMember::array(&"vortex.tensor.normalized"), - EditionMember::array(&"vortex.tensor.l2_norm"), - EditionMember::dtype(&"vortex.tensor.fixed_shape_tensor"), - EditionMember::dtype(&"vortex.tensor.vector"), - ], -}; diff --git a/vortex/src/editions/preview/v2026_06.rs b/vortex/src/editions/preview/v2026_06.rs deleted file mode 100644 index 3a888457c1f..00000000000 --- a/vortex/src/editions/preview/v2026_06.rs +++ /dev/null @@ -1,27 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! The June 2026 `preview` component cohort. - -use vortex_edition::Edition; -use vortex_edition::EditionDeclaration; -use vortex_edition::EditionId; -use vortex_edition::EditionMember; - -/// The June 2026 draft edition of the `preview` family. -pub const PREVIEW_2026_06_0: EditionId = EditionId::new("preview", 2026, 6, 0); - -/// The declaration of [`PREVIEW_2026_06_0`] and the components that join the family at it. -pub static DECLARATION: EditionDeclaration = EditionDeclaration { - edition: Edition { - id: PREVIEW_2026_06_0, - min_vortex_version: None, - }, - added: &[ - EditionMember::layout(&"vortex.list"), - // Written only by CUDA-enabled sessions, which register the layout through - // `vortex_cuda::layout::register_cuda_layout`. A writer resolves layouts against the - // enabled editions, so the GPU flat layout has to be a member to be written at all. - EditionMember::layout(&"vortex.cuda_flat"), - ], -}; diff --git a/vortex/src/editions/tests.rs b/vortex/src/editions/tests.rs index 4ba621baf7a..f3cbedc9ffb 100644 --- a/vortex/src/editions/tests.rs +++ b/vortex/src/editions/tests.rs @@ -1,22 +1,20 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::sync::Arc; - use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::array_session; use vortex_array::arrays::ChunkedArray; +use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; -use vortex_array::field_path; +use vortex_array::extension::datetime::Date; +use vortex_array::extension::datetime::TimeUnit; use vortex_array::session::ArraySessionExt; -use vortex_array::stream::ArrayStreamExt; -use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_buffer::ByteBufferMut; use vortex_edition::ComponentKind; use vortex_edition::Edition; @@ -34,14 +32,10 @@ use vortex_file::OpenOptionsSessionExt; use vortex_file::WriteOptionsSessionExt; use vortex_file::WriteStrategyBuilder; use vortex_io::session::RuntimeSession; -use vortex_layout::LayoutStrategy; -use vortex_layout::layouts::compressed::CompressingStrategy; -use vortex_layout::layouts::flat::writer::FlatLayoutStrategy; use vortex_layout::session::LayoutSession; use vortex_sequence::Sequence; use vortex_session::VortexSession; use vortex_session::registry::Id; -use vortex_utils::aliases::hash_set::HashSet; use super::CORE_2025_05_0; use super::CORE_2026_08_0; @@ -51,10 +45,13 @@ use super::CORE_2026_08_3; use super::DEFAULT_CORE_EDITION; use super::DEFAULT_PREVIEW_EDITION; use super::EDITION_DECLARATIONS; -use super::PREVIEW_2026_06_0; +use super::PREVIEW_2026_08_0; fn session() -> Result { let session = EditionSession::empty(); + for family in super::EDITION_FAMILIES { + session.declare_family(family)?; + } for declaration in EDITION_DECLARATIONS { session.declare(declaration)?; } @@ -70,55 +67,6 @@ fn every_declared_edition_validates() -> Result<(), EditionError> { Ok(()) } -/// The full encoding set of the newest frozen `core` edition. This set is frozen: the only -/// way it may change is by declaring a *new* edition, so a failure here means a frozen -/// declaration was edited. -#[test] -fn core_2026_08_1_encoding_set_is_pinned() { - let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); - let encodings = session.components_in(&CORE_2026_08_1, ComponentKind::Array); - let ids: Vec<&str> = encodings - .iter() - .map(|inclusion| inclusion.component_id.as_str()) - .collect(); - assert_eq!( - ids, - [ - "fastlanes.bitpacked", - "fastlanes.for", - "fastlanes.rle", - "vortex.alp", - "vortex.alprd", - "vortex.bool", - "vortex.bytebool", - "vortex.chunked", - "vortex.constant", - "vortex.datetimeparts", - "vortex.decimal", - "vortex.decimal_byte_parts", - "vortex.dict", - "vortex.ext", - "vortex.fixed_size_list", - "vortex.fsst", - "vortex.list", - "vortex.listview", - "vortex.masked", - "vortex.null", - "vortex.onpair", - "vortex.pco", - "vortex.primitive", - "vortex.runend", - "vortex.sequence", - "vortex.sparse", - "vortex.struct", - "vortex.varbin", - "vortex.varbinview", - "vortex.zigzag", - "vortex.zstd", - ] - ); -} - #[test] fn core_2026_08_1_dtype_set_is_pinned() { let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); @@ -131,10 +79,10 @@ fn core_2026_08_1_dtype_set_is_pinned() { } #[test] -fn core_2026_08_2_is_draft() { +fn core_2026_08_2_is_frozen() { let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); assert!( - session + !session .find(&CORE_2026_08_2) .unwrap_or_else(|| panic!("{CORE_2026_08_2} is not registered")) .is_draft() @@ -154,10 +102,10 @@ fn core_2026_08_2_is_draft() { } #[test] -fn core_2026_08_3_adds_variants() { +fn core_2026_08_3_is_frozen_and_adds_variants() { let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); assert!( - session + !session .find(&CORE_2026_08_3) .unwrap_or_else(|| panic!("{CORE_2026_08_3} is not registered")) .is_draft() @@ -189,34 +137,23 @@ fn core_2026_08_3_adds_variants() { } #[test] -fn encodings_in_editions_unions_families() { +fn preview_starts_empty() { let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); - let core_only: Vec<_> = session - .components_in(&CORE_2026_08_1, ComponentKind::Array) - .into_iter() - .map(|inclusion| inclusion.component_id) - .collect(); - let mut both = core_only.clone(); - both.extend( - session - .components_in(&PREVIEW_2026_06_0, ComponentKind::Array) - .into_iter() - .map(|inclusion| inclusion.component_id), - ); - both.sort_unstable(); - both.dedup(); - - assert!(both.len() > core_only.len()); - assert!(both.iter().any(|id| id.as_str() == "fastlanes.delta")); - assert!(both.iter().any(|id| id.as_str() == "vortex.zstd_buffers")); - assert!(core_only.iter().all(|id| both.contains(id))); + for kind in [ + ComponentKind::Array, + ComponentKind::Layout, + ComponentKind::DType, + ComponentKind::Aggregate, + ] { + assert!(session.components_in(&PREVIEW_2026_08_0, kind).is_empty()); + } } #[test] fn earlier_editions_are_subsets() { let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); let first = session.components_in(&CORE_2025_05_0, ComponentKind::Array); - let latest = session.components_in(&CORE_2026_08_1, ComponentKind::Array); + let latest = session.components_in(&DEFAULT_CORE_EDITION, ComponentKind::Array); assert!(first.iter().all(|inclusion| { latest .iter() @@ -274,6 +211,11 @@ fn default_session_enables_the_write_editions() { let session = VortexSession::default(); let enabled = session.enabled_editions().editions(); assert!(enabled.contains(&DEFAULT_CORE_EDITION)); + assert!( + session + .enabled_component_ids(ComponentKind::Array) + .contains(&Id::from("vortex.pco")) + ); #[cfg(feature = "unstable_encodings")] assert!(enabled.contains(&DEFAULT_PREVIEW_EDITION)); @@ -291,7 +233,7 @@ fn core_edition_ids_are_registered_array_encodings() { let registry = session.arrays().registry().clone(); for inclusion in session .editions() - .components_in(&CORE_2026_08_1, ComponentKind::Array) + .components_in(&DEFAULT_CORE_EDITION, ComponentKind::Array) { assert!( registry.contains_key(&inclusion.component_id), @@ -311,7 +253,7 @@ fn core_dtype_ids_are_registered_extension_dtypes() { let registry = session.dtypes().registry().clone(); for inclusion in session .editions() - .components_in(&CORE_2026_08_1, ComponentKind::DType) + .components_in(&DEFAULT_CORE_EDITION, ComponentKind::DType) { assert!( registry.contains_key(&inclusion.component_id), @@ -332,7 +274,7 @@ fn core_aggregate_ids_are_registered_aggregate_fns() { let session = VortexSession::default(); let declared = session .editions() - .components_in(&CORE_2026_08_1, ComponentKind::Aggregate); + .components_in(&DEFAULT_CORE_EDITION, ComponentKind::Aggregate); assert!( declared .iter() @@ -402,46 +344,6 @@ async fn default_session_writes_every_default_zone_aggregate() -> VortexResult<( Ok(()) } -/// Restrict arrays to the baseline core edition while allowing the modern zoned components that -/// the current default layout strategy writes. -fn baseline_core_array_session() -> VortexResult { - const SUPPORT_EDITION: EditionId = EditionId::new("writer-support", 2026, 8, 0); - static SUPPORT_DECLARATION: EditionDeclaration = EditionDeclaration { - edition: Edition { - id: SUPPORT_EDITION, - min_vortex_version: None, - }, - added: &[ - EditionMember::layout(&"vortex.zoned"), - EditionMember::aggregate(&"vortex.bounded_max"), - EditionMember::aggregate(&"vortex.bounded_min"), - EditionMember::aggregate(&"vortex.max"), - EditionMember::aggregate(&"vortex.min"), - EditionMember::aggregate(&"vortex.nan_count"), - EditionMember::aggregate(&"vortex.null_count"), - ], - }; - - let session = array_session() - .with::() - .with::() - .with::(); - vortex_file::register_default_encodings(&session); - session - .register_edition(&super::core::v2025_05::DECLARATION) - .map_err(|error| vortex_err!("{error}"))?; - session - .register_edition(&SUPPORT_DECLARATION) - .map_err(|error| vortex_err!("{error}"))?; - session - .enable_edition(CORE_2025_05_0) - .map_err(|error| vortex_err!("{error}"))?; - session - .enable_edition(SUPPORT_EDITION) - .map_err(|error| vortex_err!("{error}"))?; - Ok(session) -} - fn sequential_integers() -> PrimitiveArray { PrimitiveArray::from_iter(0..65_536i32) } @@ -451,7 +353,7 @@ const WRITER_TEST_EDITION: EditionId = EditionId::new("writer-test", 2026, 7, 0) static WRITER_TEST_DECLARATION: EditionDeclaration = EditionDeclaration { edition: Edition { id: WRITER_TEST_EDITION, - min_vortex_version: None, + min_library_version: None, }, added: &[ EditionMember::array(&"vortex.chunked"), @@ -489,6 +391,49 @@ fn writer_test_session() -> VortexResult { Ok(session) } +#[tokio::test] +async fn disabling_editions_allows_uneditioned_components() -> VortexResult<()> { + let enforced_session = array_session() + .with::() + .with::() + .with::(); + vortex_file::register_default_encodings(&enforced_session); + let array = ExtensionArray::try_new( + Date::new(TimeUnit::Days, Nullability::NonNullable).erased(), + sequential_integers().into_array(), + )? + .into_array(); + + let mut rejected = ByteBufferMut::empty(); + let error = enforced_session + .write_options() + .write(&mut rejected, array.clone().to_array_stream()) + .await + .err() + .ok_or_else(|| { + vortex_err!("writer with no enabled editions accepted an extension dtype") + })?; + assert!( + error + .to_string() + .contains("Extension DType vortex.date not permitted"), + "unexpected error: {error}" + ); + + let uneditioned_session = array_session() + .with::() + .with::(); + vortex_file::register_default_encodings(&uneditioned_session); + let mut buffer = ByteBufferMut::empty(); + uneditioned_session + .write_options() + .disable_editions() + .write(&mut buffer, array.to_array_stream()) + .await?; + assert!(!buffer.is_empty()); + Ok(()) +} + /// Write `array` with `session` and return the buffer, or the error the writer raised. async fn write_with(session: &VortexSession, array: ArrayRef) -> VortexResult { let mut buffer = ByteBufferMut::empty(); @@ -535,7 +480,7 @@ fn session_declaring(members: &[(ComponentKind, Id)]) -> VortexResult Arc { - Arc::new(CompressingStrategy::new( - FlatLayoutStrategy::default(), - forbidden_sequence_compressor, - )) -} - -async fn assert_round_trip_encodings_are_enabled( - session: &VortexSession, - strategy: Option>, - array: ArrayRef, -) -> VortexResult<()> { - let mut buffer = ByteBufferMut::empty(); - let write_options = match strategy { - Some(strategy) => session.write_options().with_strategy(strategy), - None => session.write_options(), - }; - if let Err(error) = write_options - .write(&mut buffer, array.to_array_stream()) - .await - { - let message = error.to_string(); - if message.contains("not permitted by ctx") - || message.contains("normalize forbids encoding") - { - return Ok(()); - } - return Err(error); - } - - let round_tripped = session - .open_options() - .open_buffer(buffer)? - .scan()? - .into_array_stream()? - .read_all() - .await?; - let actual: HashSet<_> = round_tripped - .depth_first_traversal() - .map(|array| array.encoding_id()) - .collect(); - let allowed: HashSet<_> = session - .enabled_component_ids(ComponentKind::Array) - .into_iter() - .collect(); - let mut forbidden: Vec<_> = actual.difference(&allowed).map(|id| id.as_str()).collect(); - forbidden.sort_unstable(); - if !forbidden.is_empty() { - return Err(vortex_err!( - "round-tripped array contains encodings outside {WRITER_TEST_EDITION}: {forbidden:?}" - )); - } - - Ok(()) -} - +/// The writer configures BtrBlocks from the enabled edition, so an effective but unavailable +/// encoding is skipped rather than produced and rejected during serialization. #[tokio::test] -async fn default_strategy_round_trip_uses_only_enabled_encodings() -> VortexResult<()> { +async fn btrblocks_respects_enabled_array_encodings() -> VortexResult<()> { let session = writer_test_session()?; - assert_round_trip_encodings_are_enabled(&session, None, sequential_integers().into_array()) - .await -} - -#[tokio::test] -async fn replacement_default_builder_round_trip_uses_only_enabled_encodings() -> VortexResult<()> { - let session = writer_test_session()?; - assert_round_trip_encodings_are_enabled( - &session, - Some(WriteStrategyBuilder::default().build()), - sequential_integers().into_array(), - ) - .await -} - -#[tokio::test] -async fn replacement_btrblocks_builder_round_trip_uses_only_enabled_encodings() -> VortexResult<()> -{ - let session = writer_test_session()?; - let strategy = WriteStrategyBuilder::default() - .with_btrblocks_builder(BtrBlocksCompressorBuilder::default()) - .build(); - assert_round_trip_encodings_are_enabled( - &session, - Some(strategy), - sequential_integers().into_array(), - ) - .await -} - -#[tokio::test] -async fn opaque_compressor_round_trip_uses_only_enabled_encodings() -> VortexResult<()> { - let session = writer_test_session()?; - let strategy = WriteStrategyBuilder::default() - .with_compressor(forbidden_sequence_compressor) - .build(); - assert_round_trip_encodings_are_enabled( - &session, - Some(strategy), - sequential_integers().into_array(), - ) - .await -} - -#[tokio::test] -async fn custom_flat_strategy_round_trip_uses_only_enabled_encodings() -> VortexResult<()> { - let session = writer_test_session()?; - let strategy = WriteStrategyBuilder::default() - .with_flat_strategy(Arc::new(FlatLayoutStrategy::default())) - .build(); - assert_round_trip_encodings_are_enabled( - &session, - Some(strategy), - sequential_integers().into_array(), - ) - .await + write_with(&session, sequential_integers().into_array()).await?; + Ok(()) } +/// An explicitly supplied strategy is not reconfigured by the writer. Its unsupported output is +/// still caught by the serialization context. #[tokio::test] -async fn custom_field_writer_round_trip_uses_only_enabled_encodings() -> VortexResult<()> { +async fn explicit_btrblocks_strategy_is_not_reconfigured() -> VortexResult<()> { let session = writer_test_session()?; - let strategy = WriteStrategyBuilder::default() - .with_field_writer(field_path!(values), custom_compressing_flat_strategy()) - .build(); - let array = - StructArray::from_fields(&[("values", sequential_integers().into_array())])?.into_array(); - assert_round_trip_encodings_are_enabled(&session, Some(strategy), array).await -} + let strategy = WriteStrategyBuilder::default().build(); + let mut buffer = ByteBufferMut::empty(); -#[tokio::test] -async fn replacement_strategy_round_trip_uses_only_enabled_encodings() -> VortexResult<()> { - let session = writer_test_session()?; - assert_round_trip_encodings_are_enabled( - &session, - Some(custom_compressing_flat_strategy()), - sequential_integers().into_array(), - ) - .await -} + let error = session + .write_options() + .with_strategy(strategy) + .write( + &mut buffer, + sequential_integers().into_array().to_array_stream(), + ) + .await + .err() + .ok_or_else(|| vortex_err!("explicit BtrBlocks strategy was unexpectedly reconfigured"))?; + assert!( + error + .to_string() + .contains("Serialized array ID vortex.sequence not permitted by ctx"), + "unexpected error: {error}" + ); -#[tokio::test] -async fn replacement_flat_strategy_round_trip_uses_only_enabled_encodings() -> VortexResult<()> { - let session = writer_test_session()?; - assert_round_trip_encodings_are_enabled( - &session, - Some(Arc::new(FlatLayoutStrategy::default())), - forbidden_sequence(65_536)?, - ) - .await + Ok(()) } +/// Compressors operate on the current in-memory array model and do not interpret edition wire +/// IDs. The serialization context is the final compatibility boundary and rejects a compressor +/// result whose serialized ID is not enabled. #[tokio::test] -async fn probe_compressor_round_trip_uses_only_enabled_encodings() -> VortexResult<()> { +async fn serialization_context_rejects_unsupported_compressor_output() -> VortexResult<()> { let session = writer_test_session()?; let strategy = WriteStrategyBuilder::default() - .with_probe_compressor(forbidden_sequence_compressor) + .with_compressor(forbidden_sequence_compressor) .build(); - assert_round_trip_encodings_are_enabled( - &session, - Some(strategy), - sequential_integers().into_array(), - ) - .await -} - -#[tokio::test] -async fn default_writer_filters_compressor_to_enabled_editions() -> VortexResult<()> { - let session = baseline_core_array_session()?; let mut buffer = ByteBufferMut::empty(); - session + let error = session .write_options() + .with_strategy(strategy) .write( &mut buffer, sequential_integers().into_array().to_array_stream(), ) - .await?; + .await + .err() + .ok_or_else(|| vortex_err!("Sequence unexpectedly had a permitted wire variant"))?; + assert!( + error + .to_string() + .contains("Serialized array ID vortex.sequence not permitted by ctx"), + "unexpected error: {error}" + ); Ok(()) } +/// The same compressor output is writable when its wire ID is enabled, without configuring the +/// compressor itself from the edition. #[tokio::test] -async fn configured_btrblocks_builder_uses_enabled_editions_in_either_order() -> VortexResult<()> { - let session = baseline_core_array_session()?; - let allowed: HashSet<_> = session - .enabled_component_ids(ComponentKind::Array) - .into_iter() - .collect(); - let strategies = [ - WriteStrategyBuilder::default() - .with_btrblocks_builder(BtrBlocksCompressorBuilder::default()) - .with_allow_encodings(allowed.clone()) - .build(), - WriteStrategyBuilder::default() - .with_allow_encodings(allowed) - .with_btrblocks_builder(BtrBlocksCompressorBuilder::default()) - .build(), - ]; - - for strategy in strategies { - let mut buffer = ByteBufferMut::empty(); - session - .write_options() - .with_strategy(strategy) - .write( - &mut buffer, - sequential_integers().into_array().to_array_stream(), - ) - .await?; - } - - Ok(()) -} +async fn serialization_context_accepts_supported_compressor_output() -> VortexResult<()> { + use crate::VortexSessionDefault; -#[tokio::test] -async fn opaque_compressor_cannot_write_outside_enabled_editions() -> VortexResult<()> { - let session = baseline_core_array_session()?; - let allowed = session - .enabled_component_ids(ComponentKind::Array) - .into_iter() - .collect(); + let session = VortexSession::default(); let strategy = WriteStrategyBuilder::default() - .with_compressor(BtrBlocksCompressorBuilder::default().build()) - .with_allow_encodings(allowed) + .with_compressor(forbidden_sequence_compressor) .build(); let mut buffer = ByteBufferMut::empty(); - let result = session + session .write_options() .with_strategy(strategy) .write( &mut buffer, sequential_integers().into_array().to_array_stream(), ) - .await; - let error = match result { - Ok(_) => { - return Err(vortex_err!( - "the unrestricted opaque compressor wrote an encoding outside core@2025.05" - )); - } - Err(error) => error, - }; - let message = error.to_string(); - assert!( - message.contains("normalize forbids encoding (vortex.sequence)"), - "unexpected error: {message}" - ); + .await?; Ok(()) } diff --git a/vortex/src/lib.rs b/vortex/src/lib.rs index 1d1ab1252ac..6e69e341cd0 100644 --- a/vortex/src/lib.rs +++ b/vortex/src/lib.rs @@ -149,7 +149,7 @@ pub mod compressor { pub use vortex_btrblocks::SchemeId; } -/// Vortex editions: named, frozen sets of encodings with a read-compatibility guarantee. +/// Vortex editions: versioned sets of serialized components. pub mod editions; pub mod dtype { @@ -265,6 +265,11 @@ pub mod encodings { pub use vortex_fsst::*; } + /// Parquet Variant array encoding. + pub mod parquet_variant { + pub use vortex_parquet_variant::*; + } + /// Pco numeric compression encoding. pub mod pco { pub use vortex_pco::*; @@ -317,6 +322,7 @@ impl VortexSessionDefault for VortexSession { .with::() .with::(); vortex_arrow::initialize(&session); + vortex_parquet_variant::initialize(&session); editions::register_default_editions(&session); editions::enable_default_editions(&session); diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index eae2db43413..5ac04804c95 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -22,7 +22,14 @@ test = false [dependencies] anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } +git2 = { workspace = true } prost-build = { workspace = true } +toml = { workspace = true } +vortex-edition = { workspace = true } +vortex-json = { workspace = true } +vortex-spatial = { workspace = true } +vortex-tensor = { workspace = true } +vortex-zstd = { workspace = true } xshell = { workspace = true } [lints] diff --git a/xtask/src/check_editions.rs b/xtask/src/check_editions.rs new file mode 100644 index 00000000000..aeaa2b258a7 --- /dev/null +++ b/xtask/src/check_editions.rs @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Check that frozen edition records under `vortex/editions` never change. +//! +//! A draft record carries no read-forever guarantee and is not mechanically locked by this check. +//! Once `min_library_version` is backfilled, the record captures the frozen edition's complete +//! membership and may never change again. Whether a record was frozen is read from the base +//! revision, so a change cannot unfreeze an edition and edit it in the same diff. +//! +//! A newly added record must also be newer than every edition already recorded for its +//! family: editions are only ever added going forward. Records are grouped by family, so +//! `vortex/editions/core/core2025.05.0.toml` must sit under the family its name declares. The +//! `family.toml` beside them documents the family rather than pinning a contract, so it is +//! exempt. +//! +//! Both revisions are read out of the object database, so the check sees committed state only +//! and never the working tree. + +use std::collections::BTreeMap; +use std::path::Path; + +use anyhow::Context; +use anyhow::anyhow; +use anyhow::bail; +use git2::Commit; +use git2::Delta; +use git2::DiffFindOptions; +use git2::Repository; +use git2::TreeWalkMode; +use git2::TreeWalkResult; +use toml::Table; + +use crate::generate_editions::FAMILY_FILE; +use crate::generate_editions::RECORD_DIR; + +/// A record carries this key once the edition's freeze has been documented. +const FROZEN_MARKER: &str = "min_library_version"; + +const REMEDY: &str = "\ +A frozen edition is immutable. To add component IDs, declare a NEW edition in + the family's declaration module and regenerate the records with + `cargo run -p xtask -- generate-editions`."; + +/// An edition's position in its family's chronology, from `..`. +type Chronology = (u16, u8, u8); + +/// Split a record file name into its family and its place in that family's chronology. +fn parse_name(name: &str) -> anyhow::Result<(&str, Chronology)> { + let malformed = || { + anyhow!( + "{RECORD_DIR}/{name} is not a valid record name. Records are named after the \ + edition they record, e.g. `core/core2026.08.0.toml`." + ) + }; + + let stem = name.strip_suffix(".toml").ok_or_else(malformed)?; + let split = stem + .find(|c: char| c.is_ascii_digit()) + .ok_or_else(malformed)?; + let (family, version) = stem.split_at(split); + if family.is_empty() || !family.chars().all(|c| c.is_ascii_lowercase()) { + return Err(malformed()); + } + + let parts: Vec<&str> = version.split('.').collect(); + let [year, month, version] = parts.as_slice() else { + return Err(malformed()); + }; + if year.len() != 4 || month.len() != 2 { + return Err(malformed()); + } + let chronology = ( + year.parse().map_err(|_| malformed())?, + month.parse().map_err(|_| malformed())?, + version.parse().map_err(|_| malformed())?, + ); + Ok((family, chronology)) +} + +/// Parse a record out of a commit's tree, or `None` when it holds no such file. +fn read_record(repo: &Repository, commit: &Commit, path: &str) -> anyhow::Result> { + let Ok(entry) = commit.tree()?.get_path(Path::new(path)) else { + return Ok(None); + }; + let blob = entry.to_object(repo)?.peel_to_blob()?; + let text = std::str::from_utf8(blob.content()) + .with_context(|| format!("{path} at {} is not UTF-8", commit.id()))?; + Ok(Some(text.parse::().with_context(|| { + format!("{path} at {} is not valid TOML", commit.id()) + })?)) +} + +/// The newest edition already recorded for each family at `commit`. +fn newest_recorded( + repo: &Repository, + commit: &Commit, +) -> anyhow::Result> { + let mut newest = BTreeMap::new(); + let Ok(entry) = commit.tree()?.get_path(Path::new(RECORD_DIR)) else { + return Ok(newest); + }; + let records = entry.to_object(repo)?.peel_to_tree()?; + + let mut malformed = None; + records.walk(TreeWalkMode::PreOrder, |_, entry| { + let Ok(name) = entry.name() else { + return TreeWalkResult::Ok; + }; + if !name.ends_with(".toml") || name == FAMILY_FILE { + return TreeWalkResult::Ok; + } + match parse_name(name) { + Ok((family, chronology)) => { + let slot = newest.entry(family.to_string()).or_insert(chronology); + *slot = (*slot).max(chronology); + TreeWalkResult::Ok + } + Err(error) => { + malformed = Some(error); + TreeWalkResult::Abort + } + } + })?; + match malformed { + Some(error) => Err(error), + None => Ok(newest), + } +} + +/// A frozen record may not change at all; name the fields that did. +fn check_modification(before: &Table, after: &Table, name: &str) -> Vec { + let mut changed: Vec<&str> = before + .keys() + .chain(after.keys()) + .map(String::as_str) + .filter(|key| before.get(*key) != after.get(*key)) + .collect(); + changed.sort_unstable(); + changed.dedup(); + + if changed.is_empty() { + return vec![]; + } + if changed.contains(&FROZEN_MARKER) && !after.contains_key(FROZEN_MARKER) { + return vec![format!( + "unfreezes {name}; an edition that recorded a {FROZEN_MARKER} carries a \ + read-forever guarantee and may never return to draft" + )]; + } + vec![format!( + "modifies the frozen record {name}: {}", + changed.join(", ") + )] +} + +/// A new record must extend its family's chronology, and be filed under that family. +fn check_addition( + path: &str, + record: &Table, + newest: &BTreeMap, +) -> anyhow::Result> { + let mut errors = Vec::new(); + let name = Path::new(path) + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| anyhow!("{path} has no file name"))?; + let (family, chronology) = parse_name(name)?; + + if let Some(previous) = newest.get(family) + && chronology <= *previous + { + errors.push(format!( + "adds {name}, which is not newer than the {family} edition already recorded \ + ({family}{}.{:02}.{}). Editions may only be added going forward.", + previous.0, previous.1, previous.2, + )); + } + + // A record's family decides which chronology it extends, so the directory it sits in has + // to agree with the family its name declares. + let directory = Path::new(path) + .parent() + .and_then(Path::file_name) + .and_then(|name| name.to_str()) + .unwrap_or_default(); + if directory != family { + errors.push(format!( + "adds {name} under {directory}/, but it records a {family} edition; records are \ + grouped by family" + )); + } + + // The file name is the edition's identity, so it has to agree with the content. + match record.get("edition").and_then(|edition| edition.as_str()) { + None => errors.push(format!("adds {name}, which has no `edition` field")), + Some(edition) if edition != name.trim_end_matches(".toml") => errors.push(format!( + "adds {name}, which records edition {edition:?}; the file name must be the \ + edition id" + )), + Some(_) => {} + } + match record.get("origin").and_then(|origin| origin.as_str()) { + None => errors.push(format!("adds {name}, which has no `origin` field")), + Some(origin) if origin.trim().is_empty() => { + errors.push(format!("adds {name}, which has an empty `origin` field")); + } + Some(_) => {} + } + Ok(errors) +} + +fn under_record_dir(path: Option<&Path>) -> bool { + path.is_some_and(|path| path.starts_with(RECORD_DIR)) +} + +fn is_family_record(path: Option<&Path>) -> bool { + path.is_some_and(|path| path.file_name().is_some_and(|name| name == FAMILY_FILE)) +} + +fn path_str(path: Option<&Path>) -> String { + path.map(|path| path.display().to_string()) + .unwrap_or_default() +} + +pub fn check_editions(base: &str) -> anyhow::Result<()> { + let repo = Repository::discover(".").context("opening the repository")?; + let base_tip = repo + .revparse_single(base) + .with_context(|| format!("cannot resolve {base:?} in this repository"))? + .peel_to_commit()?; + let head = repo.head()?.peel_to_commit()?; + + let merge_base = repo.merge_base(base_tip.id(), head.id()).with_context(|| { + format!( + "{base} and HEAD have no common ancestor. The checkout is probably too shallow; \ + this check needs `fetch-depth: 0`." + ) + })?; + let base_commit = repo.find_commit(merge_base)?; + + let mut diff = repo.diff_tree_to_tree(Some(&base_commit.tree()?), Some(&head.tree()?), None)?; + diff.find_similar(Some(DiffFindOptions::new().renames(true)))?; + + let newest = newest_recorded(&repo, &base_commit)?; + let mut errors = Vec::new(); + let mut added = Vec::new(); + + for delta in diff.deltas() { + let (old_path, new_path) = (delta.old_file().path(), delta.new_file().path()); + if !under_record_dir(old_path) && !under_record_dir(new_path) { + continue; + } + // The family record is documentation rather than a contract, so it stays editable. + if is_family_record(new_path) || is_family_record(old_path) { + continue; + } + + if delta.status() == Delta::Added { + added.push(path_str(new_path)); + continue; + } + + // Frozen-ness comes from the base revision, so a diff cannot unfreeze an edition and + // then edit it. Legacy draft records have no compatibility contract and are ignored. + let old = path_str(old_path); + let Some(before) = read_record(&repo, &base_commit, &old)? else { + continue; + }; + if !before.contains_key(FROZEN_MARKER) { + continue; + } + + let new = path_str(new_path); + if delta.status() == Delta::Modified { + let after = read_record(&repo, &head, &new)?.unwrap_or_default(); + let name = Path::new(&new) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(&new); + errors.extend(check_modification(&before, &after, name)); + } else { + let verb = match delta.status() { + Delta::Deleted => "deletes", + Delta::Renamed => "renames", + Delta::Copied => "copies", + Delta::Typechange => "retypes", + _ => "changes", + }; + let moved = if old == new { + old.clone() + } else { + format!("{old} -> {new}") + }; + errors.push(format!("{verb} the frozen record {moved}")); + } + } + + added.sort(); + for path in &added { + if let Some(record) = read_record(&repo, &head, path)? { + errors.extend(check_addition(path, &record, &newest)?); + } + } + + if errors.is_empty() { + println!("{RECORD_DIR} preserves every frozen record against {base}."); + return Ok(()); + } + + let listed = errors + .iter() + .map(|error| format!(" - it {error}")) + .collect::>() + .join("\n"); + bail!("This change breaks the edition records in {RECORD_DIR}:\n\n{listed}\n\n{REMEDY}"); +} diff --git a/xtask/src/generate_editions.rs b/xtask/src/generate_editions.rs new file mode 100644 index 00000000000..791728936fe --- /dev/null +++ b/xtask/src/generate_editions.rs @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Export the edition records under `vortex/editions`. +//! +//! Every declared edition gets one TOML file recording what it contains: the identifier, the +//! origin library or project, the minimum origin version whose reader supports it once frozen, +//! and its full member set. Records are grouped by +//! family — `vortex/editions/core/core2025.05.0.toml` — and collected from both the default +//! declarations in `vortex-edition` and optional-component declarations in their owning crates. +//! +//! A draft record carries no compatibility guarantee. Once `min_library_version` is backfilled to +//! document its freeze, the record carries a read-forever guarantee and may never change again. +//! CI enforces that against git history with `cargo run -p xtask -- check-editions`; this exporter +//! enforces the two rules that history cannot see, refusing to delete a record or to unfreeze one. + +use std::collections::BTreeSet; +use std::fs; +use std::path::Path; +use std::path::PathBuf; + +use anyhow::Context; +use anyhow::anyhow; +use vortex_edition::ComponentKind; +use vortex_edition::EDITION_DECLARATIONS; +use vortex_edition::EDITION_FAMILIES; +use vortex_edition::Edition; +use vortex_edition::EditionDeclaration; +use vortex_edition::EditionFamily; +use vortex_edition::EditionInclusion; +use vortex_edition::EditionSession; + +const GENERATED_BY: &str = "# Generated by `cargo run -p xtask -- generate-editions`.\n#"; + +const FROZEN_NOTE: &str = "\ +# This edition is frozen: it carries a read-forever guarantee, so this record of what it +# contains never changes again. Freezing a new edition adds a new file to this directory; +# editing or deleting a frozen one is rejected by CI."; + +const DRAFT_NOTE: &str = "\ +# This edition record has no read-forever guarantee. It may describe an evolving feature, +# a stable component awaiting adoption, or a release waiting to be cut. New capabilities advance +# to a new edition; after an origin release is known, min_library_version is backfilled to document +# its freeze. A frozen record never changes."; + +/// The file recording what a family is, beside that family's editions. +pub const FAMILY_FILE: &str = "family.toml"; + +/// The edition records, relative to the repository root. +pub const RECORD_DIR: &str = "vortex/editions"; + +/// Edition families owned by optional Vortex crates rather than the default facade. +static OPTIONAL_EDITION_FAMILIES: &[&EditionFamily] = &[ + &vortex_json::editions::FAMILY, + &vortex_spatial::editions::FAMILY, + &vortex_tensor::editions::FAMILY, + &vortex_zstd::editions::FAMILY, +]; + +/// Edition declarations owned by optional Vortex crates rather than the default facade. +static OPTIONAL_EDITION_DECLARATIONS: &[&EditionDeclaration] = &[ + &vortex_json::editions::DECLARATION, + &vortex_spatial::editions::DECLARATION, + &vortex_tensor::editions::DECLARATION, + &vortex_zstd::editions::DECLARATION, +]; + +/// Render a family's record: its name, origin, and what it is for. Unlike an edition record this +/// is documentation, not a contract, so it stays editable. +fn family_record(family: &EditionFamily) -> String { + let mut lines = vec![ + GENERATED_BY.to_string(), + "# This describes the family the editions beside it belong to.".to_string(), + String::new(), + format!("name = \"{}\"", family.name), + format!("origin = \"{}\"", family.origin), + String::new(), + "doc = \"\"\"".to_string(), + ]; + lines.extend(wrap(family.doc, 92)); + lines.extend(["\"\"\"".to_string(), String::new()]); + lines.join("\n") +} + +/// Wrap prose to a column, so a long doc reads as a paragraph rather than one endless line. +fn wrap(text: &str, width: usize) -> Vec { + let mut lines = Vec::new(); + let mut line = String::new(); + for word in text.split_whitespace() { + if !line.is_empty() && line.len() + 1 + word.len() > width { + lines.push(std::mem::take(&mut line)); + } + if !line.is_empty() { + line.push(' '); + } + line.push_str(word); + } + if !line.is_empty() { + lines.push(line); + } + lines +} + +/// Every component kind and the record key its members are listed under. +const KINDS: [(ComponentKind, &str); 4] = [ + (ComponentKind::Array, "arrays"), + (ComponentKind::Layout, "layouts"), + (ComponentKind::DType, "dtypes"), + (ComponentKind::Aggregate, "aggregates"), +]; + +/// Render one TOML list per component kind, each sorted by component id. +fn kind_lists( + lines: &mut Vec, + inclusions_of: impl Fn(ComponentKind) -> Vec, +) { + for (kind, key) in KINDS { + let mut inclusions = inclusions_of(kind); + inclusions.sort_by_key(|inclusion| inclusion.component_id); + if inclusions.is_empty() { + lines.push(format!("{key} = []")); + continue; + } + lines.push(format!("{key} = [")); + lines.extend( + inclusions + .iter() + .map(|inclusion| format!(" \"{}\",", inclusion.component_id)), + ); + lines.push("]".to_string()); + } +} + +/// Render one edition's record. Deterministic: every list is sorted by component id, so the +/// generated bytes depend only on the declarations. +fn record(session: &EditionSession, edition: &Edition, family: &EditionFamily) -> String { + let note = if edition.is_draft() { + DRAFT_NOTE + } else { + FROZEN_NOTE + }; + let mut lines = vec![ + GENERATED_BY.to_string(), + note.to_string(), + String::new(), + format!("edition = \"{}\"", edition.id), + format!("family = \"{}\"", edition.id.family), + format!("origin = \"{}\"", family.origin), + ]; + if let Some(min_library_version) = edition.min_library_version { + lines.push(format!("min_library_version = \"{min_library_version}\"")); + } + lines.extend([ + String::new(), + "# Components added by this edition.".to_string(), + "[added]".to_string(), + ]); + kind_lists(&mut lines, |kind| { + session + .components_in(&edition.id, kind) + .into_iter() + .filter(|inclusion| inclusion.since == edition.id) + .collect() + }); + lines.extend([ + String::new(), + "# The edition's full membership: the members above, plus every member of earlier" + .to_string(), + "# editions of the family.".to_string(), + "[components]".to_string(), + ]); + kind_lists(&mut lines, |kind| session.components_in(&edition.id, kind)); + lines.push(String::new()); + lines.join("\n") +} + +/// The records present on disk, as `family/edition.toml` paths relative to the record +/// directory. A record filed under the wrong family reads as a stray, which is what it is. +fn existing_records(dir: &Path) -> anyhow::Result> { + let mut records = BTreeSet::new(); + if !dir.exists() { + return Ok(records); + } + for family in fs::read_dir(dir).with_context(|| format!("reading {}", dir.display()))? { + let family = family?.path(); + if !family.is_dir() { + continue; + } + for entry in + fs::read_dir(&family).with_context(|| format!("reading {}", family.display()))? + { + let path = entry?.path(); + if path + .extension() + .is_some_and(|extension| extension == "toml") + && let Ok(relative) = path.strip_prefix(dir) + && let Some(relative) = relative.to_str() + { + records.insert(relative.to_string()); + } + } + } + Ok(records) +} + +/// A record carries a `min_library_version` once its freeze has been documented. +fn records_a_frozen_edition(contents: &str) -> bool { + contents + .lines() + .any(|line| line.starts_with("min_library_version = ")) +} + +pub fn generate_editions() -> anyhow::Result<()> { + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join(RECORD_DIR); + fs::create_dir_all(&dir).with_context(|| format!("creating {}", dir.display()))?; + + let session = EditionSession::empty(); + for family in EDITION_FAMILIES + .iter() + .copied() + .chain(OPTIONAL_EDITION_FAMILIES.iter().copied()) + { + session + .declare_family(family) + .map_err(|error| anyhow!("declaring edition families: {error}"))?; + } + for declaration in EDITION_DECLARATIONS + .iter() + .copied() + .chain(OPTIONAL_EDITION_DECLARATIONS.iter().copied()) + { + session + .declare(declaration) + .map_err(|error| anyhow!("declaring editions: {error}"))?; + } + session + .validate() + .map_err(|error| anyhow!("validating editions: {error}"))?; + + let mut expected = BTreeSet::new(); + for family in session.families() { + let relative = format!("{}/{FAMILY_FILE}", family.name); + let path = dir.join(&relative); + expected.insert(relative); + fs::create_dir_all(dir.join(family.name)) + .with_context(|| format!("creating the {} record directory", family.name))?; + fs::write(&path, family_record(&family)) + .with_context(|| format!("writing {}", path.display()))?; + } + + for edition in session.editions() { + let relative = format!("{}/{}.toml", edition.id.family, edition.id); + let path = dir.join(&relative); + expected.insert(relative); + fs::create_dir_all(dir.join(edition.id.family)) + .with_context(|| format!("creating the {} record directory", edition.id.family))?; + + // Freezing is permanent, and the record on disk is the only memory of it. Refusing + // here means unfreezing cannot be laundered through the exporter. + if edition.is_draft() + && let Ok(previous) = fs::read_to_string(&path) + && records_a_frozen_edition(&previous) + { + return Err(anyhow!( + "{} is recorded as frozen but its declaration is now a draft.\n\ + An edition that recorded a min_library_version carries a read-forever \ + guarantee and may never return to draft.", + edition.id, + )); + } + + let family = session.find_family(edition.id.family).ok_or_else(|| { + anyhow!( + "edition {} belongs to undeclared family {}", + edition.id, + edition.id.family + ) + })?; + fs::write(&path, record(&session, &edition, &family)) + .with_context(|| format!("writing {}", path.display()))?; + } + + let strays: Vec = existing_records(&dir)? + .difference(&expected) + .cloned() + .collect(); + if !strays.is_empty() { + return Err(anyhow!( + "{} has records with no declared edition behind them: {strays:?}.\n\ + A frozen edition may never be deleted; its declaration must stay in the \ + generator's edition declarations.", + dir.display(), + )); + } + + Ok(()) +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 1155ee3246a..8cc582be233 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,11 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +mod check_editions; +mod generate_editions; mod generate_fbs; mod generate_proto; use clap::Parser; +use crate::check_editions::check_editions; +use crate::generate_editions::generate_editions; use crate::generate_fbs::generate_fbs; use crate::generate_proto::generate_proto; @@ -17,19 +21,31 @@ struct Xtask { #[derive(clap::Subcommand)] enum Commands { + /// Subcommand to check that frozen edition records never change. + #[command(name = "check-editions")] + CheckEditions { + /// The revision to compare against. + #[arg(long, default_value = "origin/develop")] + base: String, + }, + /// Subcommand to regenerate the edition records under `vortex/editions`. + #[command(name = "generate-editions")] + Editions, /// Subcommand to regenerate flatbuffers language bindings for the Rust project. #[command(name = "generate-fbs")] - GenerateFlatbuffers, + Flatbuffers, /// Subcommand to regenerate protobuf language bindings for the Rust project. #[command(name = "generate-proto")] - GenerateProto, + Proto, } fn main() -> anyhow::Result<()> { let cli = Xtask::parse(); match cli.command { - Commands::GenerateFlatbuffers => generate_fbs()?, - Commands::GenerateProto => generate_proto()?, + Commands::CheckEditions { base } => check_editions(&base)?, + Commands::Editions => generate_editions()?, + Commands::Flatbuffers => generate_fbs()?, + Commands::Proto => generate_proto()?, } Ok(()) }