diff --git a/include/svs/core/data/io.h b/include/svs/core/data/io.h index 572ffa63..c4662045 100644 --- a/include/svs/core/data/io.h +++ b/include/svs/core/data/io.h @@ -50,6 +50,11 @@ struct DefaultWriteAccessor { void set(Data& data, size_t i, Span span) const { data.set_datum(i, span); } + + template + lib::VectorReader vector_reader(const Data& data) const { + return lib::VectorReader(data.dimensions()); + } }; // Similar to writing TO a dataset, when reading FROM a dataset, we also provide an option @@ -79,16 +84,13 @@ void populate_impl( } } -template void populate(std::istream& is, Data& data) { - auto accessor = DefaultWriteAccessor(); - +template +void populate(Data& data, WriteAccessor&& accessor, std::istream& is) { size_t num_vectors = data.size(); - size_t dims = data.dimensions(); - auto max_lines = Dynamic; auto nvectors = std::min(num_vectors, max_lines); - auto reader = lib::VectorReader(dims); + auto reader = accessor.vector_reader(data); for (size_t i = 0; i < nvectors; ++i) { reader.read(is); accessor.set(data, i, reader.data()); @@ -194,12 +196,31 @@ lib::lazy_result_t load_dataset(const File& file, const F& la return load_impl(detail::to_native(file), default_accessor, lazy); } +template F> +lib::lazy_result_t load_dataset( + std::istream& is, + WriteAccessor&& accessor, + const F& lazy, + size_t num_vectors, + size_t dims +) { + auto data = lazy(num_vectors, dims); + if constexpr (!is_view_type_v::allocator_type>) { + populate(data, std::forward(accessor), is); + } else { + throw ANNEXCEPTION("Trying to load a dataset with a view allocator from a " + "non-memory stream. This " + "is not supported since views are compatible only with " + "memory-mapped streams."); + } + return data; +} + template F> lib::lazy_result_t load_dataset(std::istream& is, const F& lazy, size_t num_vectors, size_t dims) { - auto data = lazy(num_vectors, dims); - populate(is, data); - return data; + auto accessor = DefaultWriteAccessor(); + return load_dataset(is, accessor, lazy, num_vectors, dims); } // Return whether or not a file is directly loadable via file-extension.