From b9d6ff6eb045a689e4826f64af6bc9fb08aedf3b Mon Sep 17 00:00:00 2001 From: Olivier Bonnaure Date: Thu, 16 Jul 2026 12:57:50 +0200 Subject: [PATCH 1/8] fix(admin): fix graph vertex-id normalization and stop physics churn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The graph explorer crashed ("Index out of bounds: 1 at 215:38") and rendered a disconnected pile of nodes on graphs whose keys contain colons — e.g. the edifice_rag code graph with ids like "external:Some.Ns" and "file:api:...:Foo.cs". _plain_id stripped the ":" cursor prefix by splitting on every ":" and keeping the last segment, which mangled colon-bearing keys down to a slashless id. That id crashed _node_entry (parts[1] out of bounds) and never matched an edge's _from/_to, so every real node was orphaned. Strip only the leading ":" prefix instead, and harden _node_entry / _fetch_vertex against slashless ids so a malformed endpoint degrades gracefully instead of throwing. Client: freeze vis-network physics once the layout stabilizes — re-running the solver only when new nodes are merged (double-click expand) — so large graphs (500 nodes, depth 4) settle and hold still instead of drifting forever. Adds a regression spec covering colon-in-key ids. Co-Authored-By: Claude Opus 4.8 (1M context) --- admin/app/controllers/graph_controller.sl | 21 +++++++++--- admin/public/js/admin-graph.js | 40 +++++++++++++++++++++-- admin/tests/graph_controller_spec.sl | 36 ++++++++++++++++++++ 3 files changed, 90 insertions(+), 7 deletions(-) diff --git a/admin/app/controllers/graph_controller.sl b/admin/app/controllers/graph_controller.sl index 4270ed72..3870ebee 100644 --- a/admin/app/controllers/graph_controller.sl +++ b/admin/app/controllers/graph_controller.sl @@ -135,6 +135,7 @@ class GraphController < Controller # so the canvas always has a root. nil when the id does not resolve. def _fetch_vertex(vertex_id) parts = vertex_id.split("/") + return nil if parts.length() < 2 result = SolidbClient.get_api(SolidbEndpoints.document(@db, parts[0], parts[1])) return nil unless result["ok"] return result["data"] @@ -211,14 +212,26 @@ class GraphController < Controller end def _node_entry(vertex_id, doc) + # Ids are normally "collection/key", but a placeholder endpoint (an edge + # _from/_to that lacks a slash) splits to a single element. Indexing past + # the end throws in Soli, so guard on length rather than relying on ??. parts = vertex_id.split("/") - return { "id": vertex_id, "key": parts[1] ?? vertex_id, "collection": parts[0], "doc": doc } + key = parts.length() > 1 ? parts[1] : vertex_id + return { "id": vertex_id, "key": key, "collection": parts[0], "doc": doc } end - # "db:people/bob" -> "people/bob"; plain ids pass through. + # "edifice_rag:soli_graph_nodes/file:api:Foo.cs" -> "soli_graph_nodes/file:api:Foo.cs". + # The cursor prefixes _id with ":"; the database name never contains "/", + # so the prefix always sits before the first slash. Keys themselves may hold + # colons (code-graph ids like "file:api:...:Foo.cs" or "external:Some.Ns"), so + # we strip ONLY the leading ":" - splitting on every ":" mangled the key + # into its last segment, producing a slashless id that both crashed + # _node_entry and never matched an edge's _from/_to. Plain ids pass through. def _plain_id(full_id) - parts = full_id.split(":") - return parts[parts.length() - 1] + head = full_id.split("/")[0] + return full_id unless head.includes?(":") + prefix_length = head.split(":")[0].length() + 1 + return full_id.substring(prefix_length, full_id.length()) end # Route context + collection lists for the toolbar: edge collections are diff --git a/admin/public/js/admin-graph.js b/admin/public/js/admin-graph.js index 84c11d4b..2a3352ca 100644 --- a/admin/public/js/admin-graph.js +++ b/admin/public/js/admin-graph.js @@ -18,6 +18,30 @@ window.AdminGraph = (function () { var docs = {}; // id -> full document (vertices and edges) var colors = {}; // collection name -> palette color var rootId = null; + var physicsFrozen = false; // solver off? big graphs never rest on their own + var settleTimer = null; // safety cap so motion always stops + + // Physics lays the graph out, then we switch it off so the canvas stops + // churning - a 500-node / depth-4 graph never reaches a true rest state, so + // without this it drifts forever. Dragging still works while frozen; merging + // new nodes (double-click expand) re-runs the solver via runPhysics(). + function freezePhysics() { + if (settleTimer) { clearTimeout(settleTimer); settleTimer = null; } + if (network && !physicsFrozen) { + physicsFrozen = true; + network.setOptions({ physics: { enabled: false } }); + } + } + + function runPhysics() { + if (!network) return; + physicsFrozen = false; + network.setOptions({ physics: { enabled: true } }); + // Fallback in case neither stabilized nor stabilizationIterationsDone fires + // after a dynamic re-enable (varies by vis-network version). + if (settleTimer) clearTimeout(settleTimer); + settleTimer = setTimeout(freezePhysics, 4000); + } function colorFor(collection) { if (!colors[collection]) { @@ -99,11 +123,18 @@ window.AdminGraph = (function () { smooth: { type: "continuous" } }, physics: { - barnesHut: { gravitationalConstant: -2600, springLength: 140, springConstant: 0.04 }, - stabilization: { iterations: 120 } + barnesHut: { gravitationalConstant: -2600, springLength: 140, springConstant: 0.04, damping: 0.3 }, + stabilization: { iterations: 200, updateInterval: 25, fit: true }, + // Stop the solver once nodes are basically at rest rather than letting + // it oscillate; freezePhysics() then pins it. Both matter at 500 nodes. + minVelocity: 1 }, interaction: { hover: true, tooltipDelay: 120 } }); + // Freeze once the layout settles (stabilized), and hard-freeze after the + // iteration budget in case a large hairball never fully settles. + network.on("stabilized", freezePhysics); + network.on("stabilizationIterationsDone", freezePhysics); network.on("click", function (params) { if (params.nodes.length > 0) inspect(params.nodes[0]); else if (params.edges.length > 0) inspect(params.edges[0]); @@ -136,9 +167,10 @@ window.AdminGraph = (function () { function merge(data) { ensureNetwork(); if (!network) { showError("vis-network failed to load (CDN unreachable?)"); return; } + var addedNode = false; (data.nodes || []).forEach(function (node) { if (node.doc) docs[node.id] = node.doc; - if (!nodeSet.get(node.id)) nodeSet.add(nodeVisual(node)); + if (!nodeSet.get(node.id)) { nodeSet.add(nodeVisual(node)); addedNode = true; } else if (node.doc) nodeSet.update(nodeVisual(node)); // placeholder got resolved }); (data.edges || []).forEach(function (edge) { @@ -147,6 +179,8 @@ window.AdminGraph = (function () { edgeSet.add({ id: edge.id, from: edge.from, to: edge.to, title: edge.id }); } }); + // New nodes need the solver to place them; re-run it, then it freezes again. + if (addedNode && physicsFrozen) runPhysics(); updateStats(); } diff --git a/admin/tests/graph_controller_spec.sl b/admin/tests/graph_controller_spec.sl index 787661a1..b7bec00b 100644 --- a/admin/tests/graph_controller_spec.sl +++ b/admin/tests/graph_controller_spec.sl @@ -128,6 +128,42 @@ describe("GraphController") do assert_not(data["ok"]) assert_contains(data["error"], "not found") end + + test("joins nodes whose keys contain colons (code-graph ids)") do + # Regression: _plain_id used to split on every ":" and keep the last + # segment, so a cursor id "db:code_nodes/external:Some.Ns" normalized to + # "Some.Ns" - a slashless id that crashed _node_entry (index out of + # bounds) and never matched an edge's _from/_to, orphaning every node. + SolidbClient.post_api(SolidbEndpoints.collections("admin_spec_graph"), { "name": "code_nodes" }) + SolidbClient.post_api(SolidbEndpoints.collections("admin_spec_graph"), { "name": "code_edges", "type": "edge" }) + SolidbClient.post_api(SolidbEndpoints.documents("admin_spec_graph", "code_nodes"), + { "_key": "file:api:Foo.cs", "name": "Foo.cs" }) + SolidbClient.post_api(SolidbEndpoints.documents("admin_spec_graph", "code_nodes"), + { "_key": "external:Some.Namespace", "name": "Some.Namespace" }) + SolidbClient.post_api(SolidbEndpoints.documents("admin_spec_graph", "code_edges"), + { "_from": "code_nodes/file:api:Foo.cs", "_to": "code_nodes/external:Some.Namespace" }) + + response = post("/databases/admin_spec_graph/graph/traverse", + { "start": "code_nodes/file:api:Foo.cs", "edge": "code_edges", + "direction": "outbound", "depth": "1" }) + assert_eq(res_status(response), 200) + data = res_json(response) + assert(data["ok"]) + node_ids = data["nodes"].map do |node| node["id"] end + # Full "collection/key" ids survive normalization and line up with the edge. + assert_contains(node_ids, "code_nodes/file:api:Foo.cs") + assert_contains(node_ids, "code_nodes/external:Some.Namespace") + # The key's last colon segment must NOT leak out as a bare, slashless id. + assert_not(node_ids.includes?("Some.Namespace")) + edge = data["edges"][0] + assert_eq(edge["from"], "code_nodes/file:api:Foo.cs") + assert_eq(edge["to"], "code_nodes/external:Some.Namespace") + # The traversed neighbor resolves to its document, not a dashed placeholder, + # and its rendered key is everything after the collection prefix. + neighbor = data["nodes"].filter do |node| node["id"] == "code_nodes/external:Some.Namespace" end + assert_not_null(neighbor[0]["doc"]) + assert_eq(neighbor[0]["key"], "external:Some.Namespace") + end end describe("POST /databases/:db/graph/demo") do From b8dd17e633127ff793d2eb5de0324134090f9eb9 Mon Sep 17 00:00:00 2001 From: Olivier Bonnaure Date: Sun, 19 Jul 2026 20:25:23 +0200 Subject: [PATCH 2/8] perf(vector): throttle index persistence and skip no-op reindexing persist_vector_indexes() re-serializes the entire index (all vectors plus the HNSW graph) into one blob, so calling it after every write batch made a bulk load O(batches x index size). Two changes cut that: - Persist at most once per 5s behind a dirty flag, with a shutdown flush via flush_all_stats() so the trailing window survives a graceful restart. Same throttle-on-write + flush-on-shutdown model already used for collection stats. A hard crash can lose at most one window, rebuildable from the documents' embedding fields. - On UPDATE, compare the extracted f32 vectors and skip the delete+reinsert when every index's embedding is unchanged. An incremental graph sync that only rewrites metadata now pays no HNSW churn and doesn't dirty the index into a full re-serialize. Adds coverage for batch-write durability across reopen and for the skip-reindex path. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/storage/collection/core.rs | 63 +++++++++++++++++ src/storage/collection/crud.rs | 50 ++++++++------ src/storage/collection/mod.rs | 6 ++ src/storage/collection/vector.rs | 32 ++++++++- src/storage/engine.rs | 8 ++- tests/vector_index_tests.rs | 114 +++++++++++++++++++++++++++++++ 6 files changed, 249 insertions(+), 24 deletions(-) diff --git a/src/storage/collection/core.rs b/src/storage/collection/core.rs index 6f2f5fb2..7886bda9 100644 --- a/src/storage/collection/core.rs +++ b/src/storage/collection/core.rs @@ -7,6 +7,11 @@ use parking_lot::RwLock; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::Arc; +/// Minimum seconds between throttled vector-index persists during bulk writes. +/// The trailing window is made durable by the shutdown flush +/// (`flush_vector_indexes` via the engine's flush-all). +const VEC_PERSIST_THROTTLE_SECS: u64 = 5; + impl Collection { /// Create a new collection handle pub fn new(name: String, db: Arc) -> Self { @@ -57,6 +62,8 @@ impl Collection { chunk_count: Arc::new(AtomicUsize::new(chunk_count)), count_dirty: Arc::new(AtomicBool::new(false)), last_flush_time: Arc::new(std::sync::atomic::AtomicU64::new(0)), + vec_dirty: Arc::new(AtomicBool::new(false)), + vec_last_persist: Arc::new(std::sync::atomic::AtomicU64::new(0)), change_sender: Arc::new(change_sender), collection_type: Arc::new(RwLock::new(collection_type)), bloom_filters: Arc::new(DashMap::new()), @@ -128,6 +135,62 @@ impl Collection { } } + /// Persist vector indexes to disk, but at most once per second and only when + /// there are unpersisted changes. + /// + /// `persist_vector_indexes()` re-serializes the *entire* index (all vectors + + /// the HNSW graph) into a single blob, so calling it after every write batch + /// during a bulk load is O(batches × index size) — the dominant cost when a + /// large embedding-bearing collection is (re)loaded. Throttling collapses that + /// burst to roughly one persist per second. The trailing window is made + /// durable by `flush_vector_indexes()` on shutdown — the same + /// throttle-on-write + flush-on-shutdown model already used for collection + /// stats (`flush_stats` / `flush_stats_throttled`). A hard crash can lose at + /// most the last window of index updates, which are recoverable by rebuilding + /// the index from the documents' embedding fields. + pub fn persist_vector_indexes_throttled(&self) { + if !self.vec_dirty.load(Ordering::Relaxed) { + return; // Nothing to persist + } + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + // Persist at most once per VEC_PERSIST_THROTTLE_SECS. This interval is + // deliberately larger than one second: a single write batch against a + // big index can itself take >1s (full-index re-serialize), so a 1s + // window would still persist on every batch and defeat the throttle. + if now.saturating_sub(self.vec_last_persist.load(Ordering::Relaxed)) + < VEC_PERSIST_THROTTLE_SECS + { + return; + } + self.flush_vector_indexes(); + } + + /// Persist vector indexes to disk if there are unpersisted changes, + /// regardless of throttle. Called on shutdown (via the engine's flush-all) + /// so the trailing throttle window can't be lost across a graceful restart. + pub fn flush_vector_indexes(&self) { + // Claim the dirty flag up front so a writer that dirties again after we + // snapshot the index isn't wrongly cleared (mirrors `flush_stats`). + if !self.vec_dirty.swap(false, Ordering::Relaxed) { + return; // Nothing to persist + } + if let Err(e) = self.persist_vector_indexes() { + tracing::warn!("Failed to persist vector indexes: {}", e); + // Re-arm so a later throttled call / shutdown flush retries rather + // than silently dropping the change. + self.vec_dirty.store(true, Ordering::Relaxed); + return; + } + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + self.vec_last_persist.store(now, Ordering::Relaxed); + } + /// Compact the collection to remove tombstones and reclaim space pub fn compact(&self) { if let Some(cf) = self.db.cf_handle(&self.name) { diff --git a/src/storage/collection/crud.rs b/src/storage/collection/crud.rs index e9b34fcc..d3bdcf93 100644 --- a/src/storage/collection/crud.rs +++ b/src/storage/collection/crud.rs @@ -265,9 +265,14 @@ impl Collection { db.write(&batch) .map_err(|e| DbError::InternalError(format!("Failed to update document: {}", e)))?; - // Update vector indexes in-memory (separate from WriteBatch) - self.update_vector_indexes_on_delete(key); - self.update_vector_indexes_on_upsert(key, &new_value); + // Update vector indexes in-memory (separate from WriteBatch). Skip the + // delete+reinsert entirely when the embedding is unchanged — a document + // rewritten for non-embedding fields must not pay HNSW churn or dirty + // the index (which would trigger a full re-serialize on persist). + if !self.vector_index_unchanged(&old_value, &new_value) { + self.update_vector_indexes_on_delete(key); + self.update_vector_indexes_on_upsert(key, &new_value); + } // Enforce version retention (best-effort, off the atomic write). if versioned { @@ -392,9 +397,14 @@ impl Collection { }); } - // Update vector indexes in-memory (separate from WriteBatch) - self.update_vector_indexes_on_delete(key); - self.update_vector_indexes_on_upsert(key, &new_value); + // Update vector indexes in-memory (separate from WriteBatch). Skip the + // delete+reinsert entirely when the embedding is unchanged — a document + // rewritten for non-embedding fields must not pay HNSW churn or dirty + // the index (which would trigger a full re-serialize on persist). + if !self.vector_index_unchanged(&old_value, &new_value) { + self.update_vector_indexes_on_delete(key); + self.update_vector_indexes_on_upsert(key, &new_value); + } // Enforce version retention (best-effort, off the atomic write). if versioned { @@ -576,9 +586,7 @@ impl Collection { self.update_vector_indexes_on_upsert(key, doc_value); } // Persist vector indexes after batch - if let Err(e) = self.persist_vector_indexes() { - tracing::warn!("Failed to persist vector indexes: {}", e); - } + self.persist_vector_indexes_throttled(); Ok(count) } @@ -659,9 +667,7 @@ impl Collection { } // Persist vector indexes after batch delete - if let Err(e) = self.persist_vector_indexes() { - tracing::warn!("Failed to persist vector indexes: {}", e); - } + self.persist_vector_indexes_throttled(); // Commit batch atomically: all document deletions + index removals together db.write(&batch) @@ -803,9 +809,15 @@ impl Collection { batch.put_cf(&cf, entry_key, Vec::new()); } - // Update vector indexes in-memory (separate from WriteBatch) - self.update_vector_indexes_on_delete(key); - self.update_vector_indexes_on_upsert(key, &new_value); + // Update vector indexes in-memory (separate from + // WriteBatch). Skip the delete+reinsert when the embedding is + // unchanged so a bulk update that only rewrites metadata + // (e.g. an incremental graph sync) pays no HNSW churn and + // doesn't dirty the index into a full re-serialize. + if !self.vector_index_unchanged(&old_value, &new_value) { + self.update_vector_indexes_on_delete(key); + self.update_vector_indexes_on_upsert(key, &new_value); + } change_events.push((key.clone(), old_value, new_value)); updated_docs.push(doc); @@ -818,9 +830,7 @@ impl Collection { } // Persist vector indexes after batch update - if let Err(e) = self.persist_vector_indexes() { - tracing::warn!("Failed to persist vector indexes: {}", e); - } + self.persist_vector_indexes_throttled(); // Commit batch atomically: all document updates + index updates together db.write(&batch) @@ -958,9 +968,7 @@ impl Collection { self.update_vector_indexes_on_upsert(key, doc_value); } // Persist vector indexes after batch - if let Err(e) = self.persist_vector_indexes() { - tracing::warn!("Failed to persist vector indexes: {}", e); - } + self.persist_vector_indexes_throttled(); // Update document count let count = inserted_docs.len(); diff --git a/src/storage/collection/mod.rs b/src/storage/collection/mod.rs index 6a1dbd5a..51eea34f 100644 --- a/src/storage/collection/mod.rs +++ b/src/storage/collection/mod.rs @@ -138,6 +138,10 @@ pub struct Collection { pub(crate) count_dirty: Arc, /// Last flush time in seconds since UNIX epoch (for throttling) pub(crate) last_flush_time: Arc, + /// Whether in-memory vector indexes have changes not yet persisted to disk + pub(crate) vec_dirty: Arc, + /// Last vector-index persist time in seconds since UNIX epoch (throttling) + pub(crate) vec_last_persist: Arc, /// Broadcast channel for real-time change events pub change_sender: Arc>, /// Collection type (document, edge, blob) @@ -163,6 +167,8 @@ impl Clone for Collection { chunk_count: self.chunk_count.clone(), count_dirty: self.count_dirty.clone(), last_flush_time: self.last_flush_time.clone(), + vec_dirty: self.vec_dirty.clone(), + vec_last_persist: self.vec_last_persist.clone(), change_sender: self.change_sender.clone(), collection_type: self.collection_type.clone(), bloom_filters: self.bloom_filters.clone(), diff --git a/src/storage/collection/vector.rs b/src/storage/collection/vector.rs index d1d6bc4e..7d203b6e 100644 --- a/src/storage/collection/vector.rs +++ b/src/storage/collection/vector.rs @@ -410,6 +410,11 @@ impl Collection { self.extract_vector(doc_value, &config.field, config.dimension) { let _ = index.insert(doc_key, &vector); + // Mark for (throttled) persistence; the actual disk write is + // deferred so a bulk load doesn't re-serialize the whole + // index per batch. + self.vec_dirty + .store(true, std::sync::atomic::Ordering::Relaxed); if config.embedding_source.is_some() { // Vector is now present; drop any stale pending-embed marker. self.clear_embed_pending(&config.name, doc_key); @@ -436,8 +441,15 @@ impl Collection { /// Update vector indexes on doc delete pub(crate) fn update_vector_indexes_on_delete(&self, doc_key: &str) { + let mut changed = false; for entry in self.vector_indexes.iter() { - let _ = entry.remove(doc_key); + if let Ok(true) = entry.remove(doc_key) { + changed = true; + } + } + if changed { + self.vec_dirty + .store(true, std::sync::atomic::Ordering::Relaxed); } // Drop any pending-embed markers for this doc across all auto-embed indexes. for config in self.get_all_vector_index_configs() { @@ -447,6 +459,24 @@ impl Collection { } } + /// Whether a document UPDATE leaves every vector index's embedding + /// unchanged. When true, the caller can skip the delete+reinsert into the + /// vector index entirely — no HNSW churn and no dirty flag (hence no + /// persist). This is the dominant case for an incremental graph sync, which + /// rewrites node documents to refresh metadata / a content hash while their + /// embeddings stay byte-identical. Compared on the extracted `f32` vectors, + /// so it matches exactly what would otherwise be re-indexed. + pub(crate) fn vector_index_unchanged(&self, old_value: &Value, new_value: &Value) -> bool { + for config in self.get_all_vector_index_configs() { + let old_vec = self.extract_vector(old_value, &config.field, config.dimension); + let new_vec = self.extract_vector(new_value, &config.field, config.dimension); + if old_vec != new_vec { + return false; + } + } + true + } + /// Helper to extract vector from document pub(crate) fn extract_vector( &self, diff --git a/src/storage/engine.rs b/src/storage/engine.rs index 82b758a3..fc291ff9 100644 --- a/src/storage/engine.rs +++ b/src/storage/engine.rs @@ -381,8 +381,9 @@ impl StorageEngine { } } - /// Flush all collection stats to disk - /// Called on shutdown to ensure counts are persisted + /// Flush all collection stats and vector indexes to disk. + /// Called on shutdown to ensure counts and the throttled vector-index + /// persistence window are durable across a graceful restart. pub fn flush_all_stats(&self) { let databases = self.list_databases(); @@ -392,6 +393,9 @@ impl StorageEngine { for coll_name in collections { if let Ok(collection) = database.get_collection(&coll_name) { collection.flush_stats(); + // Persist any vector-index changes that the per-write + // throttle deferred (see `persist_vector_indexes_throttled`). + collection.flush_vector_indexes(); } } } diff --git a/tests/vector_index_tests.rs b/tests/vector_index_tests.rs index 278c8766..24911e6a 100644 --- a/tests/vector_index_tests.rs +++ b/tests/vector_index_tests.rs @@ -760,3 +760,117 @@ fn test_vector_index_delete_and_search() { "Second closest 'b' should now be first" ); } + +// ============================================================================ +// Throttled-persist durability (batch write path -> shutdown flush -> reopen) +// ============================================================================ + +#[test] +fn test_vector_index_batch_write_survives_flush_and_reopen() { + // The batch write paths now persist the vector index on a throttle rather + // than after every batch (avoiding a full re-serialize per chunk on a bulk + // load). Durability of the deferred window is provided by the shutdown + // flush. This exercises that whole contract end to end. + let tmp = TempDir::new().expect("temp dir"); + let path = tmp.path().to_str().unwrap().to_string(); + + { + let engine = StorageEngine::new(&path).expect("engine"); + engine + .create_collection("products".to_string(), None) + .unwrap(); + let collection = engine.get_collection("products").unwrap(); + collection + .create_vector_index(VectorIndexConfig::new( + "embedding_idx".to_string(), + "embedding".to_string(), + 3, + )) + .unwrap(); + + // Bulk write path: marks the index dirty and defers the disk persist. + collection + .insert_batch(vec![ + json!({"_key": "d1", "embedding": [1.0, 0.0, 0.0]}), + json!({"_key": "d2", "embedding": [0.0, 1.0, 0.0]}), + json!({"_key": "d3", "embedding": [0.0, 0.0, 1.0]}), + ]) + .unwrap(); + + // In-memory index must be correct immediately after the batch, even if + // the disk persist was throttled out. + let live = collection + .vector_search("embedding_idx", &[1.0, 0.0, 0.0], 1, None) + .unwrap(); + assert_eq!(live.len(), 1); + assert_eq!(live[0].doc_key, "d1"); + + // The shutdown flush makes the deferred state durable. + collection.flush_vector_indexes(); + } + + // Reopen at the same path: the index must load from disk with the vectors. + { + let engine = StorageEngine::new(&path).expect("reopen engine"); + let collection = engine.get_collection("products").unwrap(); + let hits = collection + .vector_search("embedding_idx", &[0.0, 1.0, 0.0], 1, None) + .expect("search after reopen"); + assert_eq!(hits.len(), 1, "index should have survived reopen"); + assert_eq!(hits[0].doc_key, "d2"); + } +} + +#[test] +fn test_vector_update_skips_reindex_when_embedding_unchanged() { + // A document UPDATE that leaves the embedding untouched must not disturb the + // vector index (the fast path that avoids HNSW churn), while an UPDATE that + // changes the embedding must re-index correctly. + let (engine, _tmp) = create_test_engine(); + engine + .create_collection("products".to_string(), None) + .unwrap(); + let coll = engine.get_collection("products").unwrap(); + coll.create_vector_index(VectorIndexConfig::new( + "embedding_idx".to_string(), + "embedding".to_string(), + 3, + )) + .unwrap(); + + coll.insert(json!({"_key": "d1", "embedding": [1.0, 0.0, 0.0], "label": "a"})) + .unwrap(); + coll.insert(json!({"_key": "d2", "embedding": [0.0, 1.0, 0.0], "label": "b"})) + .unwrap(); + + // Non-embedding field change: vector unchanged, index must still hold it. + coll.update("d1", json!({"label": "a2"})).unwrap(); + let hits = coll + .vector_search("embedding_idx", &[1.0, 0.0, 0.0], 1, None) + .unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!( + hits[0].doc_key, "d1", + "unchanged-embedding update must not drop the vector" + ); + + // Embedding change: index must reflect the new vector. + coll.update("d1", json!({"embedding": [0.0, 0.0, 1.0]})) + .unwrap(); + let z = coll + .vector_search("embedding_idx", &[0.0, 0.0, 1.0], 1, None) + .unwrap(); + assert_eq!(z[0].doc_key, "d1", "changed embedding must be re-indexed"); + let x = coll + .vector_search("embedding_idx", &[1.0, 0.0, 0.0], 2, None) + .unwrap(); + let d1_score = x + .iter() + .find(|r| r.doc_key == "d1") + .map(|r| r.score) + .unwrap(); + assert!( + d1_score < 0.5, + "d1 should have moved off its old direction; score={d1_score}" + ); +} From ed77562018845b7e4fe332d5200e7b77b947a572 Mon Sep 17 00:00:00 2001 From: Olivier Bonnaure Date: Sun, 19 Jul 2026 20:25:40 +0200 Subject: [PATCH 3/8] feat(build): support Windows x86_64 and move TLS to rustls Publishes solidb-windows-amd64.zip alongside the existing Linux/macOS tarballs. The server core needed no porting - TCP listeners, protocol multiplexing, RocksDB paths and the test suite were already portable, and the Unix-only pieces (daemonize, signals, 0600 modes) were already cfg-gated. The blockers were native C dependencies and distribution. TLS moves from native-tls to rustls, which removes OpenSSL from the graph entirely (cargo tree -i openssl-sys now matches nothing) and with it the Perl/NASM/openssl-src build requirement on Windows: - Drop the vendored openssl dep. It was referenced by no Rust code; it only existed to force vendoring onto the openssl-sys that reqwest and tokio-tungstenite pulled in. - reqwest 0.13 and tokio-tungstenite move to rustls. clients/rust-client is a dev-dependency of solidb, so it moves too - otherwise openssl stays in cargo test. - The queue's dev HTTP client drops danger_accept_invalid_hostnames, which only exists on the native-tls backend. Behavior is preserved: under rustls danger_accept_invalid_certs already skips hostname verification. - jsonwebtoken keeps the aws_lc_rs backend (7320477 chose it to drop the vulnerable rsa crate, which the rust_crypto alternative reintroduces). A windows-gated aws-lc-rs with prebuilt-nasm removes the NASM requirement; CMake and MSVC are already on the runner image. - Docker no longer installs libssl3. CI gains the x86_64-pc-windows-msvc release target with LLVM (librocksdb-sys bindgen needs libclang) and zip packaging, plus a windows-check compile job so portability breakage surfaces on PRs rather than at tag time. Also switches sdbql_join_tests off hardcoded /tmp paths to TempDir; they never cleaned up, so they leaked a RocksDB directory per run. Known Windows gaps, documented in the README: --daemon is Unix-only and exits with an error, and .admin_password is written without the owner-only ACL used on Unix. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 68 ++++++++++++++++++++++++++++++++-- Cargo.lock | 41 ++++++++++---------- Cargo.toml | 29 ++++++++++++--- Dockerfile | 3 +- README.md | 31 +++++++++++++++- clients/rust-client/Cargo.toml | 11 +++++- install.sh | 7 ++++ src/main.rs | 8 +++- src/queue/mod.rs | 5 ++- tests/sdbql_join_tests.rs | 56 +++++++++++++++------------- 10 files changed, 197 insertions(+), 62 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8446a519..7dd3a9e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: - name: Install system dependencies run: | sudo apt-get update - sudo apt-get install -y libssl-dev pkg-config + sudo apt-get install -y pkg-config - name: Install Rust uses: dtolnay/rust-toolchain@stable @@ -46,6 +46,37 @@ jobs: - name: Run tests run: cargo test --release + # Compile-only gate for Windows. The full test suite is not run here: a cold RocksDB C++ + # build on windows-latest is slow enough to dominate CI, and the value of this job is + # catching portability breakage on the PR rather than at tag time in build-binaries. + windows-check: + runs-on: windows-latest + steps: + - uses: actions/checkout@v5 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Install LLVM + uses: KyleMayes/install-llvm-action@v2 + with: + version: "18.1" + env: true + + - name: Cargo cache + uses: actions/cache@v5 + with: + path: | + ~/.cargo/registry/ + ~/.cargo/git/ + target/ + key: ${{ runner.os }}-cargo-check-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo-check- + + - name: Check + run: cargo check --all-targets + clippy: runs-on: ubuntu-latest steps: @@ -54,7 +85,7 @@ jobs: - name: Install system dependencies run: | sudo apt-get update - sudo apt-get install -y libssl-dev pkg-config + sudo apt-get install -y pkg-config - name: Install Rust uses: dtolnay/rust-toolchain@stable @@ -114,6 +145,9 @@ jobs: - target: aarch64-unknown-linux-gnu os: ubuntu-latest artifact: solidb-linux-arm64 + - target: x86_64-pc-windows-msvc + os: windows-latest + artifact: solidb-windows-amd64 runs-on: ${{ matrix.os }} permissions: contents: write @@ -129,7 +163,7 @@ jobs: if: runner.os == 'Linux' run: | sudo apt-get update - sudo apt-get install -y libssl-dev pkg-config + sudo apt-get install -y pkg-config - name: Install system dependencies (Linux ARM64 cross) if: matrix.target == 'aarch64-unknown-linux-gnu' @@ -137,18 +171,44 @@ jobs: sudo apt-get update sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu + # librocksdb-sys runs bindgen, which needs libclang. MSVC and CMake are already on + # the windows-latest image; LLVM is not. + - name: Install LLVM (Windows) + if: runner.os == 'Windows' + uses: KyleMayes/install-llvm-action@v2 + with: + version: "18.1" + env: true + - name: Build run: cargo build --release --target ${{ matrix.target }} - name: Package binaries + if: runner.os != 'Windows' run: | tar -czf ${{ matrix.artifact }}.tar.gz -C target/${{ matrix.target }}/release solidb solidb-dump solidb-restore + - name: Package binaries (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $rel = "target/${{ matrix.target }}/release" + Compress-Archive -Path "$rel/solidb.exe","$rel/solidb-dump.exe","$rel/solidb-restore.exe" ` + -DestinationPath "${{ matrix.artifact }}.zip" + - name: Upload to release + if: runner.os != 'Windows' env: GH_TOKEN: ${{ github.token }} run: gh release upload --clobber ${{ github.ref_name }} ${{ matrix.artifact }}.tar.gz + - name: Upload to release (Windows) + if: runner.os == 'Windows' + shell: pwsh + env: + GH_TOKEN: ${{ github.token }} + run: gh release upload --clobber ${{ github.ref_name }} ${{ matrix.artifact }}.zip + publish-release: needs: [build-binaries] if: startsWith(github.ref, 'refs/tags/v') @@ -209,7 +269,7 @@ jobs: - name: Install system dependencies run: | sudo apt-get update - sudo apt-get install -y libssl-dev pkg-config + sudo apt-get install -y pkg-config - name: Install Rust uses: dtolnay/rust-toolchain@stable diff --git a/Cargo.lock b/Cargo.lock index a6cdea96..8c2c9bd8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1919,6 +1919,7 @@ dependencies = [ "hyper", "hyper-util", "rustls", + "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls", @@ -2979,15 +2980,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" -[[package]] -name = "openssl-src" -version = "300.6.0+3.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8e8cbfd3a4a8c8f089147fd7aaa33cf8c7450c4d09f8f80698a0cf093abeff4" -dependencies = [ - "cc", -] - [[package]] name = "openssl-sys" version = "0.9.112" @@ -2996,7 +2988,6 @@ checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb" dependencies = [ "cc", "libc", - "openssl-src", "pkg-config", "vcpkg", ] @@ -3347,9 +3338,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.9" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" dependencies = [ "bytes", "cfg_aliases", @@ -3389,16 +3380,16 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.14" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" dependencies = [ "cfg_aliases", "libc", "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -3727,6 +3718,9 @@ dependencies = [ "native-tls", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", "rustls-pki-types", "serde", "serde_json", @@ -3734,6 +3728,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", + "tokio-rustls", "tower", "tower-http", "tower-service", @@ -3942,6 +3937,7 @@ checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ "aws-lc-rs", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -4420,6 +4416,7 @@ dependencies = [ "argon2", "async-stream", "async-trait", + "aws-lc-rs", "axum", "base32", "base64", @@ -4455,7 +4452,6 @@ dependencies = [ "nanoid", "notify", "once_cell", - "openssl", "opentelemetry", "parking_lot", "prometheus", @@ -4847,9 +4843,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" dependencies = [ "tinyvec_macros", ] @@ -4938,9 +4934,11 @@ checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" dependencies = [ "futures-util", "log", - "native-tls", + "rustls", + "rustls-native-certs", + "rustls-pki-types", "tokio", - "tokio-native-tls", + "tokio-rustls", "tungstenite", ] @@ -5172,8 +5170,9 @@ dependencies = [ "http", "httparse", "log", - "native-tls", "rand 0.9.2", + "rustls", + "rustls-pki-types", "sha1", "thiserror 2.0.18", "utf-8", diff --git a/Cargo.toml b/Cargo.toml index a8d0df8b..9e60afb2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,7 +29,7 @@ async-trait = "0.1" # WebSocket Client (for global changefeed aggregation) -tokio-tungstenite = { version = "0.28", features = ["native-tls"] } +tokio-tungstenite = { version = "0.28", features = ["rustls-tls-native-roots"] } url = "2.4" # JSON serialization @@ -76,6 +76,9 @@ cron = "0.15" # Authentication argon2 = "0.5" +# Keep the aws_lc_rs backend: the `rust_crypto` alternative pulls in the `rsa` crate, which +# commit 7320477 deliberately removed (RUSTSEC-2023-0071, no fixed version). See the +# windows-gated aws-lc-rs entry below for how this stays buildable on windows-msvc. jsonwebtoken = { version = "10.3", default-features = false, features = ["aws_lc_rs", "use_pem"] } once_cell = "1.19" rand_core = "0.6" @@ -91,10 +94,18 @@ subtle = "2.6" tempfile = "3.10" # HTTP client (for benchmarks) -reqwest = { version = "0.13", features = ["json", "blocking", "multipart"] } - -# Vendored OpenSSL for cross-compilation support -openssl = { version = "0.10", features = ["vendored"] } +# rustls rather than the default native-tls: it keeps the build free of OpenSSL, which +# otherwise needs Perl + NASM on Windows. default-features = false drops http2 and charset +# along with native-tls, so both are re-added explicitly. +reqwest = { version = "0.13", default-features = false, features = [ + "json", + "blocking", + "multipart", + "rustls", + "http2", + "charset", + "system-proxy", +] } # FUSE filesystem (requires macFUSE on macOS) fuser = { version = "0.16", optional = true } @@ -191,6 +202,14 @@ solidb-client = { path = "clients/rust-client" } [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = "0.6" +# aws-lc-sys (pulled in by jsonwebtoken and by reqwest's rustls provider) assembles its +# x86_64 asm with NASM on windows-msvc, which is not on the GitHub runner image. The +# prebuilt-nasm feature uses the object files shipped in the crate instead, leaving CMake + +# MSVC as the only build requirements — both preinstalled on windows-latest. Feature +# unification applies this to every aws-lc-rs in the graph. +[target.'cfg(windows)'.dependencies] +aws-lc-rs = { version = "1", features = ["prebuilt-nasm"] } + [features] fuse = ["dep:fuser"] diff --git a/Dockerfile b/Dockerfile index 86a500e5..9f1733e1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,9 +14,10 @@ LABEL org.opencontainers.image.vendor="Solisoft" LABEL org.opencontainers.image.licenses="MIT" # Install runtime dependencies +# libssl3 is no longer needed: TLS is rustls, statically linked into the binary. +# ca-certificates still is — rustls reads the system trust store. RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ - libssl3 \ libzstd1 \ curl \ && rm -rf /var/lib/apt/lists/* \ diff --git a/README.md b/README.md index fa8a5e93..91bdaca3 100644 --- a/README.md +++ b/README.md @@ -87,17 +87,44 @@ The server starts on `http://localhost:6745` with a web dashboard. > **Note**: A default admin user is created on startup with a randomly generated password displayed in the logs. +## 💻 Platforms + +Prebuilt binaries are published for each release: + +| Platform | Artifact | +|---|---| +| Linux x86_64 | `solidb-linux-amd64.tar.gz` | +| Linux arm64 | `solidb-linux-arm64.tar.gz` | +| macOS arm64 | `solidb-darwin-arm64.tar.gz` | +| Windows x86_64 | `solidb-windows-amd64.zip` | + +On Linux and macOS, `install.sh` fetches and installs the right one. On Windows, extract the +`.zip` and add the folder to your `PATH`, then run `solidb.exe --port 6745 --data-dir .\data`. + +Two Windows caveats: `--daemon` is Unix-only and exits with an error — run SoliDB in a +console, or wrap it with a service manager such as [NSSM](https://nssm.cc) or `sc.exe`. And +the generated `.admin_password` file under the data directory is written with default ACLs +rather than the owner-only permissions used on Unix, so restrict the data directory yourself +on a shared machine. + ## 📋 Build Requirements ### Ubuntu/Debian ```bash -sudo apt-get install -y build-essential clang libclang-dev pkg-config libssl-dev libzstd-dev +sudo apt-get install -y build-essential clang libclang-dev pkg-config libzstd-dev ``` ### Arch Linux ```bash -sudo pacman -S base-devel clang gcc pkg-config openssl zstd +sudo pacman -S base-devel clang gcc pkg-config zstd +``` + +### Windows +Requires the MSVC build tools, CMake, and LLVM (RocksDB's bindgen needs `libclang`): +```powershell +winget install Microsoft.VisualStudio.2022.BuildTools Kitware.CMake LLVM.LLVM ``` +Expect a long first build — RocksDB is a large C++ tree. ## 📚 Learn More diff --git a/clients/rust-client/Cargo.toml b/clients/rust-client/Cargo.toml index 4d5fd79f..2c9a04e9 100644 --- a/clients/rust-client/Cargo.toml +++ b/clients/rust-client/Cargo.toml @@ -23,7 +23,16 @@ serde_json = "1.0" rmp-serde = "1.3" thiserror = "2.0" chrono = { version = "0.4", features = ["serde"] } -reqwest = { version = "0.12", features = ["json"] } +# rustls rather than the default native-tls: this crate is a dev-dependency of solidb, so +# native-tls here would drag openssl-sys back into `cargo test` and require a system OpenSSL +# on Windows. No native-tls-specific API is used. +reqwest = { version = "0.12", default-features = false, features = [ + "json", + "rustls-tls-native-roots", + "http2", + "charset", + "system-proxy", +] } # Offline sync dependencies rusqlite = { version = "0.31", features = ["bundled", "chrono", "serde_json", "uuid"] } uuid = { version = "1.7", features = ["v4", "serde"] } diff --git a/install.sh b/install.sh index 9a4afa97..82f675c5 100755 --- a/install.sh +++ b/install.sh @@ -27,6 +27,13 @@ OS="$(uname -s)" case "$OS" in Linux*) OS="linux" ;; Darwin*) OS="darwin" ;; + MINGW*|MSYS*|CYGWIN*) + echo "Error: this installer does not support Windows." + echo "Download solidb-windows-amd64.zip from:" + echo " https://github.com/solisoft/solidb/releases/latest" + echo "then extract it and add the folder to your PATH." + exit 1 + ;; *) echo "Error: unsupported operating system: $OS"; exit 1 ;; esac diff --git a/src/main.rs b/src/main.rs index e6cbfe80..925ff885 100644 --- a/src/main.rs +++ b/src/main.rs @@ -265,8 +265,12 @@ async fn async_main(args: Args) -> anyhow::Result<()> { if !args.peers.is_empty() && cluster_config.keyfile.is_none() { anyhow::bail!( "Cluster peers are configured but no keyfile is available. \ - Create a shared secret (e.g. `openssl rand -hex 32 > solidb.key`, \ - same file on every node) and pass it with --keyfile. \ + Create a shared secret of 32 cryptographically random bytes, hex-encoded, \ + and pass it with --keyfile (the same file on every node). \ + Unix: `openssl rand -hex 32 > solidb.key`. PowerShell: \ + `$b=[byte[]]::new(32);[Security.Cryptography.RandomNumberGenerator]::Fill($b);\ + [BitConverter]::ToString($b).Replace('-','').ToLower() \ + | Out-File -Encoding ascii solidb.key`. \ Refusing to start an unauthenticated cluster." ); } diff --git a/src/queue/mod.rs b/src/queue/mod.rs index 1428c398..a2818174 100644 --- a/src/queue/mod.rs +++ b/src/queue/mod.rs @@ -52,10 +52,13 @@ impl QueueWorker { .build() .expect("reqwest::Client builds with defaults"); + // Under rustls, danger_accept_invalid_certs(true) installs a verifier that skips + // hostname verification as well as chain validation, so it alone is equivalent to the + // native-tls certs+hostnames pair this used to set. danger_accept_invalid_hostnames + // does not exist outside the native-tls backend — do not re-add it. let dev_http_client = reqwest::Client::builder() .timeout(Duration::from_secs(30)) .danger_accept_invalid_certs(true) - .danger_accept_invalid_hostnames(true) .build() .expect("reqwest::Client builds with permissive TLS"); diff --git a/tests/sdbql_join_tests.rs b/tests/sdbql_join_tests.rs index f84b9bf2..d1372e2e 100644 --- a/tests/sdbql_join_tests.rs +++ b/tests/sdbql_join_tests.rs @@ -3,14 +3,17 @@ use serde_json::json; use solidb::error::DbResult; use solidb::sdbql::{parse, QueryExecutor}; use solidb::storage::StorageEngine; +use tempfile::TempDir; use uuid::Uuid; -// Helper to create a test storage engine with sample data -fn setup_test_data() -> DbResult<(StorageEngine, String)> { - // Use unique paths and database names to avoid conflicts +// Helper to create a test storage engine with sample data. +// The returned TempDir owns the on-disk database and must be kept alive for as long as the +// engine is used — dropping it deletes the directory. +fn setup_test_data() -> DbResult<(StorageEngine, String, TempDir)> { + // Use unique database names to avoid conflicts let uuid = Uuid::new_v4(); - let db_path = format!("/tmp/test_join_db_{}", uuid); - let storage = StorageEngine::new(&db_path)?; + let tmp = TempDir::new().expect("Failed to create temp dir"); + let storage = StorageEngine::new(tmp.path().to_str().unwrap())?; let db_name = format!("test_db_{}", uuid); storage.create_database(db_name.clone())?; @@ -39,7 +42,7 @@ fn setup_test_data() -> DbResult<(StorageEngine, String)> { profiles_coll.insert(json!({"_key": "p2", "user_key": "u2", "bio": "Designer"}))?; // Note: u3 (Charlie) has no profile - Ok((storage, db_name)) + Ok((storage, db_name, tmp)) } #[test] @@ -89,7 +92,7 @@ fn test_parse_multiple_joins() -> DbResult<()> { #[test] fn test_execute_basic_inner_join() -> DbResult<()> { - let (storage, db_name) = setup_test_data()?; + let (storage, db_name, _tmp) = setup_test_data()?; let executor = QueryExecutor::with_database(&storage, db_name); let query_str = r#" @@ -118,7 +121,7 @@ fn test_execute_basic_inner_join() -> DbResult<()> { #[test] fn test_execute_left_join() -> DbResult<()> { - let (storage, db_name) = setup_test_data()?; + let (storage, db_name, _tmp) = setup_test_data()?; let executor = QueryExecutor::with_database(&storage, db_name); let query_str = r#" @@ -150,7 +153,7 @@ fn test_execute_left_join() -> DbResult<()> { #[test] fn test_multiple_joins() -> DbResult<()> { - let (storage, db_name) = setup_test_data()?; + let (storage, db_name, _tmp) = setup_test_data()?; let executor = QueryExecutor::with_database(&storage, db_name); let query_str = r#" @@ -186,7 +189,7 @@ fn test_multiple_joins() -> DbResult<()> { #[test] fn test_join_with_filter() -> DbResult<()> { - let (storage, db_name) = setup_test_data()?; + let (storage, db_name, _tmp) = setup_test_data()?; let executor = QueryExecutor::with_database(&storage, db_name); let query_str = r#" @@ -210,7 +213,7 @@ fn test_join_with_filter() -> DbResult<()> { #[test] fn test_join_with_complex_condition() -> DbResult<()> { - let (storage, db_name) = setup_test_data()?; + let (storage, db_name, _tmp) = setup_test_data()?; let executor = QueryExecutor::with_database(&storage, db_name); let query_str = r#" @@ -237,7 +240,7 @@ fn test_join_with_complex_condition() -> DbResult<()> { #[test] fn test_join_with_aggregation() -> DbResult<()> { - let (storage, db_name) = setup_test_data()?; + let (storage, db_name, _tmp) = setup_test_data()?; let executor = QueryExecutor::with_database(&storage, db_name); let query_str = r#" @@ -266,7 +269,8 @@ fn test_join_with_aggregation() -> DbResult<()> { #[test] fn test_join_empty_collection() -> DbResult<()> { let uuid = Uuid::new_v4(); - let storage = StorageEngine::new(format!("/tmp/test_join_empty_db_{}", uuid))?; + let _tmp = TempDir::new().expect("Failed to create temp dir"); + let storage = StorageEngine::new(_tmp.path().to_str().unwrap())?; let db_name = format!("empty_test_{}", uuid); @@ -341,7 +345,7 @@ fn test_parse_full_join_without_outer() -> DbResult<()> { #[test] fn test_execute_right_join() -> DbResult<()> { - let (storage, db_name) = setup_test_data()?; + let (storage, db_name, _tmp) = setup_test_data()?; let executor = QueryExecutor::with_database(&storage, db_name); let query_str = r#" @@ -367,7 +371,7 @@ fn test_execute_right_join() -> DbResult<()> { #[test] fn test_execute_full_outer_join() -> DbResult<()> { - let (storage, db_name) = setup_test_data()?; + let (storage, db_name, _tmp) = setup_test_data()?; let executor = QueryExecutor::with_database(&storage, db_name); let query_str = r#" @@ -402,7 +406,8 @@ fn test_execute_full_outer_join() -> DbResult<()> { #[test] fn test_right_join_with_no_left_matches() -> DbResult<()> { let uuid = Uuid::new_v4(); - let storage = StorageEngine::new(format!("/tmp/test_right_join_db_{}", uuid))?; + let _tmp = TempDir::new().expect("Failed to create temp dir"); + let storage = StorageEngine::new(_tmp.path().to_str().unwrap())?; let db_name = format!("right_test_{}", uuid); @@ -431,7 +436,8 @@ fn test_right_join_with_no_left_matches() -> DbResult<()> { #[test] fn test_full_outer_join_comprehensive() -> DbResult<()> { let uuid = Uuid::new_v4(); - let storage = StorageEngine::new(format!("/tmp/test_full_outer_db_{}", uuid))?; + let _tmp = TempDir::new().expect("Failed to create temp dir"); + let storage = StorageEngine::new(_tmp.path().to_str().unwrap())?; let db_name = format!("full_test_{}", uuid); @@ -495,10 +501,10 @@ fn test_full_outer_join_comprehensive() -> DbResult<()> { // edge cases where hash-key equality must match values_equal semantics) // ============================================================================ -fn setup_hash_join_data() -> DbResult<(StorageEngine, String)> { +fn setup_hash_join_data() -> DbResult<(StorageEngine, String, TempDir)> { let uuid = Uuid::new_v4(); - let db_path = format!("/tmp/test_hashjoin_db_{}", uuid); - let storage = StorageEngine::new(&db_path)?; + let tmp = TempDir::new().expect("Failed to create temp dir"); + let storage = StorageEngine::new(tmp.path().to_str().unwrap())?; let db_name = format!("test_db_{}", uuid); storage.create_database(db_name.clone())?; storage.create_collection(format!("{}:left_docs", db_name), None)?; @@ -516,7 +522,7 @@ fn setup_hash_join_data() -> DbResult<(StorageEngine, String)> { right.insert(json!({"_key": "r3", "val": null, "flag": true}))?; // null matches null right.insert(json!({"_key": "r4", "val": 99, "flag": true}))?; // matches nothing - Ok((storage, db_name)) + Ok((storage, db_name, tmp)) } /// Int and float forms of the same number must join (values_equal compares @@ -524,7 +530,7 @@ fn setup_hash_join_data() -> DbResult<(StorageEngine, String)> { /// reads as null on both sides. #[test] fn test_hash_join_number_normalization_and_null() -> DbResult<()> { - let (storage, db_name) = setup_hash_join_data()?; + let (storage, db_name, _tmp) = setup_hash_join_data()?; let executor = QueryExecutor::with_database(&storage, db_name); let query = parse( @@ -545,7 +551,7 @@ fn test_hash_join_number_normalization_and_null() -> DbResult<()> { /// the remaining conjuncts per candidate. #[test] fn test_hash_join_with_residual_condition() -> DbResult<()> { - let (storage, db_name) = setup_hash_join_data()?; + let (storage, db_name, _tmp) = setup_hash_join_data()?; let executor = QueryExecutor::with_database(&storage, db_name); let query = parse( @@ -566,7 +572,7 @@ fn test_hash_join_with_residual_condition() -> DbResult<()> { /// nested loop with identical results. #[test] fn test_join_non_equi_fallback() -> DbResult<()> { - let (storage, db_name) = setup_hash_join_data()?; + let (storage, db_name, _tmp) = setup_hash_join_data()?; let executor = QueryExecutor::with_database(&storage, db_name); let query = parse( @@ -586,7 +592,7 @@ fn test_join_non_equi_fallback() -> DbResult<()> { /// match array. #[test] fn test_hash_left_join_keeps_unmatched() -> DbResult<()> { - let (storage, db_name) = setup_hash_join_data()?; + let (storage, db_name, _tmp) = setup_hash_join_data()?; let executor = QueryExecutor::with_database(&storage, db_name); let query = parse( From 5ad35c37924f1b6b994b3f80bd93076d9e7013b6 Mon Sep 17 00:00:00 2001 From: Olivier Bonnaure Date: Mon, 20 Jul 2026 08:51:54 +0200 Subject: [PATCH 4/8] docs(vector): correct throttle interval in persist doc comment The doc comment said "at most once per second" while VEC_PERSIST_THROTTLE_SECS is 5, contradicting the comment a few lines below that explains why a 1s window would defeat the throttle. Reference the constant instead of restating a number that can drift. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/storage/collection/core.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/storage/collection/core.rs b/src/storage/collection/core.rs index 7886bda9..3458d5b6 100644 --- a/src/storage/collection/core.rs +++ b/src/storage/collection/core.rs @@ -135,14 +135,14 @@ impl Collection { } } - /// Persist vector indexes to disk, but at most once per second and only when - /// there are unpersisted changes. + /// Persist vector indexes to disk, but at most once per + /// `VEC_PERSIST_THROTTLE_SECS` and only when there are unpersisted changes. /// /// `persist_vector_indexes()` re-serializes the *entire* index (all vectors + /// the HNSW graph) into a single blob, so calling it after every write batch /// during a bulk load is O(batches × index size) — the dominant cost when a /// large embedding-bearing collection is (re)loaded. Throttling collapses that - /// burst to roughly one persist per second. The trailing window is made + /// burst to roughly one persist per window. The trailing window is made /// durable by `flush_vector_indexes()` on shutdown — the same /// throttle-on-write + flush-on-shutdown model already used for collection /// stats (`flush_stats` / `flush_stats_throttled`). A hard crash can lose at From 555aadaa32db037725121509de4d920e0baa698b Mon Sep 17 00:00:00 2001 From: Olivier Bonnaure Date: Sun, 19 Jul 2026 20:26:25 +0200 Subject: [PATCH 5/8] chore: release v0.32.0 Windows x86_64 builds, the rustls TLS migration, and the vector-index persistence throttle. Keeps the docs-site version pill in sync with Cargo.toml per CLAUDE.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 18 ++++++++++++++++++ Cargo.lock | 2 +- Cargo.toml | 2 +- doc/app/views/home/index.html.slv | 2 +- 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6eaf6af0..71f8098e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## [0.32.0](https://github.com/solisoft/solidb/compare/v0.31.0...v0.32.0) (2026-07-19) + +### Features + +* **Windows x86_64 builds**: releases now include `solidb-windows-amd64.zip` (`solidb.exe`, `solidb-dump.exe`, `solidb-restore.exe`) alongside the Linux and macOS tarballs. Two caveats for operators: + * `--daemon` is Unix-only and exits with an error. Run SoliDB in a console, or wrap it with a service manager such as NSSM or `sc.exe`. + * The generated `.admin_password` file is written with default ACLs rather than the owner-only permissions used on Unix. Restrict the data directory yourself on a shared machine. + * FUSE (`solidb-fuse`) remains Unix-only, and `solidb update` still does not support Windows. + +### Changes + +* **TLS moves from OpenSSL to rustls.** OpenSSL is no longer in the dependency graph at all. The Docker image no longer installs `libssl3`, and building from source no longer needs `libssl-dev` (or Perl/NASM on Windows). Certificate validation now uses the platform trust store via rustls rather than OpenSSL; `ca-certificates` is still required in the container image. + +### Performance + +* Vector-index persistence is throttled to at most once per 5s behind a dirty flag, with a shutdown flush, instead of re-serializing the whole index (all vectors + HNSW graph) after every write batch. Bulk loads into embedding-bearing collections are no longer O(batches × index size). +* Document updates that leave every vector index's embedding unchanged skip the delete+reinsert entirely, so an incremental sync that only rewrites metadata pays no HNSW churn. + ## [0.26.4](https://github.com/solisoft/solidb/compare/v0.26.3...v0.26.4) (2026-06-11) ### Security diff --git a/Cargo.lock b/Cargo.lock index 8c2c9bd8..8c75c1f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4410,7 +4410,7 @@ dependencies = [ [[package]] name = "solidb" -version = "0.31.0" +version = "0.32.0" dependencies = [ "anyhow", "argon2", diff --git a/Cargo.toml b/Cargo.toml index 9e60afb2..e57a42d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["clients/rust-client", "sdbql-core", "benchmarks"] [package] name = "solidb" -version = "0.31.0" +version = "0.32.0" edition = "2021" default-run = "solidb" description = "A lightweight, high-performance structured database server written in Rust." diff --git a/doc/app/views/home/index.html.slv b/doc/app/views/home/index.html.slv index c2e53d50..ca7cfe0f 100644 --- a/doc/app/views/home/index.html.slv +++ b/doc/app/views/home/index.html.slv @@ -25,7 +25,7 @@
- v0.31.0 + v0.32.0 GitHub From 5e0be17fd181bc742340299872ebaabc914e7cfd Mon Sep 17 00:00:00 2001 From: Olivier Bonnaure Date: Mon, 20 Jul 2026 09:26:26 +0200 Subject: [PATCH 6/8] fix(ci): build RocksDB with cl.exe, not clang++, on Windows install-llvm-action with env:true exported CC/CXX=clang++. Because the target is x86_64-pc-windows-msvc, cc-rs emits MSVC-style flags, which plain clang++ rejects: clang++: error: unknown argument: '-EHsc' clang++: error: unknown argument: '-std:c++20' librocksdb-sys only needs libclang for bindgen, not a compiler override, and LLVM is already preinstalled on windows-latest. Set LIBCLANG_PATH and leave CC/CXX unset so RocksDB is compiled by cl.exe. Also drops the LLVM download from both Windows jobs. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7dd3a9e8..dda70006 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,11 +57,11 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable - - name: Install LLVM - uses: KyleMayes/install-llvm-action@v2 - with: - version: "18.1" - env: true + # See the note in build-binaries: point bindgen at the preinstalled libclang, but + # leave CC/CXX alone so RocksDB is built by cl.exe rather than clang++. + - name: Configure libclang + shell: pwsh + run: echo "LIBCLANG_PATH=C:\Program Files\LLVM\bin" >> $env:GITHUB_ENV - name: Cargo cache uses: actions/cache@v5 @@ -171,14 +171,15 @@ jobs: sudo apt-get update sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu - # librocksdb-sys runs bindgen, which needs libclang. MSVC and CMake are already on - # the windows-latest image; LLVM is not. - - name: Install LLVM (Windows) + # librocksdb-sys runs bindgen, which needs libclang. LLVM, MSVC and CMake are all + # preinstalled on windows-latest, so this only points bindgen at libclang.dll. + # Do NOT export CC/CXX=clang here: the target is *-pc-windows-msvc, so cc-rs emits + # MSVC-style flags (-EHsc, -std:c++20) that plain clang++ rejects. RocksDB must be + # compiled by cl.exe. + - name: Configure libclang (Windows) if: runner.os == 'Windows' - uses: KyleMayes/install-llvm-action@v2 - with: - version: "18.1" - env: true + shell: pwsh + run: echo "LIBCLANG_PATH=C:\Program Files\LLVM\bin" >> $env:GITHUB_ENV - name: Build run: cargo build --release --target ${{ matrix.target }} From c6a34d472ce71995bddffcebc4a1cca7b208dcdc Mon Sep 17 00:00:00 2001 From: Olivier Bonnaure Date: Fri, 24 Jul 2026 15:44:02 +0200 Subject: [PATCH 7/8] fix(restore): honour --database/--collection overrides Both flags are documented as overrides ("use instead of name in dump"), but all three restore paths resolved the target as record.get("_database").or_else(|| args.database.clone()) so the name embedded in the dump always won and the flag was only consulted when the dump carried no name. Every dump emits `_database` and `_collection`, so the fallback was unreachable and both flags were dead: `solidb-restore -d staging --input prod.dump` silently restored into `prod`. Flip the precedence in process_index_record, process_blob_chunk and process_doc so the CLI flag wins and the dump name is the fallback. Verified end to end against a throwaway instance: a dump carrying `_database: fixture_src` restored with `-d fixture_dst` creates only fixture_dst (documents, collection and index records all follow the override); fixture_src is never created. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/bin/solidb-restore.rs | 89 +++++++++++++++++++++++++-------------- 1 file changed, 57 insertions(+), 32 deletions(-) diff --git a/src/bin/solidb-restore.rs b/src/bin/solidb-restore.rs index 62e156b9..542c6a9e 100644 --- a/src/bin/solidb-restore.rs +++ b/src/bin/solidb-restore.rs @@ -415,18 +415,27 @@ async fn process_index_record( total_imported: &mut u64, total_failed: &mut u64, ) -> Result<(), Box> { - let db = record - .get("_database") - .and_then(|s| s.as_str()) - .map(|s| s.to_string()) - .or_else(|| args.database.clone()) + // CLI flags are overrides: they win over the names embedded in the dump. + let db = args + .database + .clone() + .or_else(|| { + record + .get("_database") + .and_then(|s| s.as_str()) + .map(|s| s.to_string()) + }) .ok_or("No database specified in index record or args")?; - let coll = record - .get("_collection") - .and_then(|s| s.as_str()) - .map(|s| s.to_string()) - .or_else(|| args.collection.clone()) + let coll = args + .collection + .clone() + .or_else(|| { + record + .get("_collection") + .and_then(|s| s.as_str()) + .map(|s| s.to_string()) + }) .ok_or("No collection specified in index record or args")?; // Make sure DB and collection exist before posting the index @@ -547,19 +556,28 @@ async fn process_blob_chunk( total_imported: &mut u64, total_failed: &mut u64, ) -> Result<(), Box> { - // Determine target DB and Collection from header - let db = header_doc - .get("_database") - .and_then(|s| s.as_str()) - .map(|s| s.to_string()) - .or_else(|| args.database.clone()) + // Determine target DB and Collection from header. + // CLI flags are overrides: they win over the names embedded in the dump. + let db = args + .database + .clone() + .or_else(|| { + header_doc + .get("_database") + .and_then(|s| s.as_str()) + .map(|s| s.to_string()) + }) .ok_or("No database specified in chunk or args")?; - let coll = header_doc - .get("_collection") - .and_then(|s| s.as_str()) - .map(|s| s.to_string()) - .or_else(|| args.collection.clone()) + let coll = args + .collection + .clone() + .or_else(|| { + header_doc + .get("_collection") + .and_then(|s| s.as_str()) + .map(|s| s.to_string()) + }) .ok_or("No collection specified in chunk or args")?; // Create DB/Collection if needed @@ -649,19 +667,26 @@ async fn process_doc( total_imported: &mut u64, total_failed: &mut u64, ) -> Result<(), Box> { - // Determine target DB and Collection - let db = doc - .get("_database") - .and_then(|s| s.as_str()) - .map(|s| s.to_string()) - .or_else(|| args.database.clone()) + // Determine target DB and Collection. + // CLI flags are overrides: they win over the names embedded in the dump. + let db = args + .database + .clone() + .or_else(|| { + doc.get("_database") + .and_then(|s| s.as_str()) + .map(|s| s.to_string()) + }) .ok_or("No database specified in doc or args")?; - let coll = doc - .get("_collection") - .and_then(|s| s.as_str()) - .map(|s| s.to_string()) - .or_else(|| args.collection.clone()) + let coll = args + .collection + .clone() + .or_else(|| { + doc.get("_collection") + .and_then(|s| s.as_str()) + .map(|s| s.to_string()) + }) .ok_or("No collection specified in doc or args")?; // Create DB/Collection if needed From 8d9b384eaf8744cc5afc1442b5dc5f1603d6ef33 Mon Sep 17 00:00:00 2001 From: Olivier Bonnaure Date: Fri, 24 Jul 2026 15:45:05 +0200 Subject: [PATCH 8/8] chore: release v0.32.1 Patch release for the solidb-restore --database/--collection override fix. Keeps the docs-site version pill in sync with Cargo.toml per CLAUDE.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 6 ++++++ Cargo.lock | 2 +- Cargo.toml | 2 +- doc/app/views/home/index.html.slv | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71f8098e..b5d7aec5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [0.32.1](https://github.com/solisoft/solidb/compare/v0.32.0...v0.32.1) (2026-07-24) + +### Fixes + +* **`solidb-restore` now honours `--database` / `--collection`.** Both flags are documented as overrides, but the target was resolved as "name embedded in the dump, falling back to the flag". Since every dump emits `_database` and `_collection`, the fallback was unreachable and both flags were silently ignored — `solidb-restore -d staging --input prod.dump` restored into `prod`. The CLI flag now wins and the dump's name is the fallback, across document, blob-chunk and index records. **Operators who relied on the old behaviour to restore a dump back into its original database can simply omit `-d`.** + ## [0.32.0](https://github.com/solisoft/solidb/compare/v0.31.0...v0.32.0) (2026-07-19) ### Features diff --git a/Cargo.lock b/Cargo.lock index 8c75c1f9..e924cc92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4410,7 +4410,7 @@ dependencies = [ [[package]] name = "solidb" -version = "0.32.0" +version = "0.32.1" dependencies = [ "anyhow", "argon2", diff --git a/Cargo.toml b/Cargo.toml index e57a42d1..fa752d77 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["clients/rust-client", "sdbql-core", "benchmarks"] [package] name = "solidb" -version = "0.32.0" +version = "0.32.1" edition = "2021" default-run = "solidb" description = "A lightweight, high-performance structured database server written in Rust." diff --git a/doc/app/views/home/index.html.slv b/doc/app/views/home/index.html.slv index ca7cfe0f..0fcc08b2 100644 --- a/doc/app/views/home/index.html.slv +++ b/doc/app/views/home/index.html.slv @@ -25,7 +25,7 @@
- v0.32.0 + v0.32.1 GitHub