diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 516614f..4c28a88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,16 +29,20 @@ jobs: strategy: matrix: include: - - cc: gcc-12 + - id: gcc-12 + cc: gcc-12 cxx: g++-12 - - cc: clang + - id: clang + cc: clang cxx: clang++ - name: ${{ matrix.cc }} + name: Build (${{ matrix.cc }}) env: CC: ${{ matrix.cc }} CXX: ${{ matrix.cxx }} steps: - uses: actions/checkout@v4 + - name: Install Julia + run: curl -fsSL https://install.julialang.org | sh -s -- -y - name: CMake run: | sudo apt-get update @@ -46,6 +50,41 @@ jobs: cmake -B build - name: Build run: VERBOSE=true make -C build -j `nproc` + - name: Archive Build + run: tar -czf build.tar.gz build + - uses: actions/upload-artifact@v4 + with: + name: compliance-build-${{ matrix.id }} + path: build.tar.gz + retention-days: 1 + + test: + name: Test (${{ matrix.cc }}) + needs: build + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + matrix: + include: + - id: gcc-12 + cc: gcc-12 + - id: clang + cc: clang + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: compliance-build-${{ matrix.id }} + - name: Restore Build + run: tar -xzf build.tar.gz + - name: Install Runtime Dependencies + run: | + sudo apt-get update + sudo apt-get install libhdf5-dev + - uses: prefix-dev/setup-pixi@v0.10.0 + with: + run-install: false + cache: false - name: Matrix Tests run: ctest --test-dir ./build/test/bash - name: Install Julia @@ -56,3 +95,7 @@ jobs: run: julia -e 'using Pkg; Pkg.add(["Finch", "HDF5"])' - name: Tensor Tests run: ctest --test-dir ./build/test/julia + - name: Compliance Tests + env: + BINSPARSE_BUILD_DIR: ${{ github.workspace }}/build + run: ./test/compliance/run-binsparse-tests.sh -q diff --git a/CMakeLists.txt b/CMakeLists.txt index 1776473..3bf70c9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,6 +48,22 @@ FetchContent_Declare( ) FetchContent_MakeAvailable(cJSON) +FetchContent_Declare( + semver_c + GIT_REPOSITORY https://github.com/h2non/semver.c.git + GIT_TAG v1.0.0 +) +FetchContent_MakeAvailable(semver_c) +add_library(binsparse_semver OBJECT ${semver_c_SOURCE_DIR}/semver.c) +set_target_properties(binsparse_semver PROPERTIES + POSITION_INDEPENDENT_CODE ON) +target_include_directories(binsparse_semver PRIVATE + $) +target_sources(binsparse PRIVATE $) +target_sources(binsparse_dynamic PRIVATE $) +target_include_directories(binsparse PRIVATE ${semver_c_SOURCE_DIR}) +target_include_directories(binsparse_dynamic PRIVATE ${semver_c_SOURCE_DIR}) + # Restore original BUILD_SHARED_LIBS setting set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_BACKUP}) @@ -122,6 +138,8 @@ install(FILES if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) enable_testing() + add_subdirectory(test/compliance) + if (ENABLE_SANITIZERS) set(SANITIZER_FLAGS "-fsanitize=address,undefined") target_compile_options(binsparse INTERFACE ${SANITIZER_FLAGS} -g -O1 -fno-omit-frame-pointer) diff --git a/examples/bsp-ls.c b/examples/bsp-ls.c index 8c30e5d..a6807db 100644 --- a/examples/bsp-ls.c +++ b/examples/bsp-ls.c @@ -41,7 +41,14 @@ void print_group_info(hid_t g, const char* name) { assert(cJSON_IsString(version_)); char* version_string = cJSON_GetStringValue(version_); - // TODO: check version. + error = bsp_check_version_compatible(version_string); + if (error != BSP_SUCCESS) { + printf("Unsupported Binsparse version %s: %s\n", version_string, + bsp_get_error_string(error)); + cJSON_Delete(j); + free(json_string); + return; + } cJSON* format_ = cJSON_GetObjectItemCaseSensitive(binsparse, "format"); assert(format_ != NULL); diff --git a/include/binsparse/array.h b/include/binsparse/array.h index 0570d10..02b063c 100644 --- a/include/binsparse/array.h +++ b/include/binsparse/array.h @@ -40,9 +40,9 @@ bsp_construct_array_t_allocator(bsp_array_t* array, size_t size, size_t byte_size = size * bsp_type_size(type); array->allocator = allocator; - array->data = array->allocator.malloc(byte_size); + array->data = byte_size == 0 ? NULL : array->allocator.malloc(byte_size); - if (array->data == NULL) { + if (byte_size != 0 && array->data == NULL) { return BSP_ERROR_MEMORY; } @@ -65,7 +65,8 @@ static inline bsp_error_t bsp_copy_construct_array_t(bsp_array_t* array, return error; } - memcpy(array->data, other.data, other.size * bsp_type_size(other.type)); + if (other.size != 0) + memcpy(array->data, other.data, other.size * bsp_type_size(other.type)); return BSP_SUCCESS; } @@ -104,7 +105,9 @@ static inline bsp_error_t bsp_fp_array_to_complex(bsp_array_t* array) { } static inline void bsp_destroy_array_t(bsp_array_t* array) { - array->allocator.free(array->data); + if (array->data != NULL) { + array->allocator.free(array->data); + } } static inline bool bsp_array_equal(bsp_array_t x, bsp_array_t y) { diff --git a/include/binsparse/binsparse.h b/include/binsparse/binsparse.h index 189aa7c..fbb5f05 100644 --- a/include/binsparse/binsparse.h +++ b/include/binsparse/binsparse.h @@ -6,8 +6,6 @@ #pragma once -#define BINSPARSE_VERSION "0.1" - #include #include #include @@ -19,4 +17,5 @@ #include #include #include +#include #include diff --git a/include/binsparse/binsparse_cJSON.h b/include/binsparse/binsparse_cJSON.h index e72b065..19c2513 100644 --- a/include/binsparse/binsparse_cJSON.h +++ b/include/binsparse/binsparse_cJSON.h @@ -8,6 +8,7 @@ #include #include +#include #include @@ -25,12 +26,24 @@ bsp_error_t bsp_write_matrix_cjson(const char* fname, bsp_matrix_t matrix, const char* group, cJSON* user_json, int compression_level); +bsp_error_t bsp_write_tensor_cjson(const char* fname, bsp_tensor_t tensor, + const char* group, cJSON* user_json, + int compression_level); + #ifdef BSP_BINSPARSE_HDF5_H bsp_error_t bsp_write_matrix_to_group_cjson(hid_t f, bsp_matrix_t matrix, cJSON* user_json, int compression_level); #endif +#ifdef BSP_USE_HDF5 +#include + +bsp_error_t bsp_write_tensor_to_group_cjson(hid_t f, bsp_tensor_t tensor, + cJSON* user_json, + int compression_level); +#endif + #ifdef __cplusplus } #endif diff --git a/include/binsparse/detail/allocator.h b/include/binsparse/detail/allocator.h index 86d817a..0576a61 100644 --- a/include/binsparse/detail/allocator.h +++ b/include/binsparse/detail/allocator.h @@ -13,5 +13,4 @@ typedef struct bsp_allocator_t { void (*free)(void*); } bsp_allocator_t; -const static bsp_allocator_t bsp_default_allocator = {.malloc = malloc, - .free = free}; +const static bsp_allocator_t bsp_default_allocator = {malloc, free}; diff --git a/include/binsparse/detail/hdf5_types.h b/include/binsparse/detail/hdf5_types.h index 02f0315..0cf20c9 100644 --- a/include/binsparse/detail/hdf5_types.h +++ b/include/binsparse/detail/hdf5_types.h @@ -31,7 +31,7 @@ static inline hid_t bsp_get_hdf5_standard_type(bsp_type_t type) { } else if (type == BSP_FLOAT64) { return H5T_IEEE_F64LE; } else if (type == BSP_BINT8) { - return H5T_STD_I8LE; + return H5T_STD_U8LE; } else { return H5I_INVALID_HID; } @@ -86,7 +86,7 @@ static inline bsp_type_t bsp_get_bsp_type(hid_t type) { // way to generically determine the HDF5 native types for // stdint's fixed width integer types. static inline hid_t bsp_get_hdf5_native_type(bsp_type_t type) { - if (type == BSP_INT8 || type == BSP_BINT8) { + if (type == BSP_INT8) { if (sizeof(int8_t) == sizeof(char)) { return H5T_NATIVE_CHAR; } else if (sizeof(int8_t) == sizeof(short)) { @@ -142,7 +142,7 @@ static inline hid_t bsp_get_hdf5_native_type(bsp_type_t type) { } else { assert(false); } - } else if (type == BSP_UINT8) { + } else if (type == BSP_UINT8 || type == BSP_BINT8) { if (sizeof(uint8_t) == sizeof(unsigned char)) { return H5T_NATIVE_UCHAR; } else if (sizeof(uint8_t) == sizeof(unsigned short)) { diff --git a/include/binsparse/hdf5_wrapper.h b/include/binsparse/hdf5_wrapper.h index 4a2ce5d..bf331ef 100644 --- a/include/binsparse/hdf5_wrapper.h +++ b/include/binsparse/hdf5_wrapper.h @@ -42,19 +42,21 @@ static inline bsp_error_t bsp_write_array(hid_t f, const char* label, hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE); - // Choose 1 MiB, the default chunk cache size, as our chunk size. - size_t chunk_size = 1024 * 1024 / bsp_type_size(array.type); + if (array.size > 0) { + // Choose 1 MiB, the default chunk cache size, as our chunk size. + size_t chunk_size = 1024 * 1024 / bsp_type_size(array.type); - // If the dataset is smaller than the chunk size, cap the chunk size. - if (array.size < chunk_size) { - chunk_size = array.size; - } + // If the dataset is smaller than the chunk size, cap the chunk size. + if (array.size < chunk_size) { + chunk_size = array.size; + } - hsize[0] = chunk_size; - H5Pset_chunk(dcpl, 1, hsize); + hsize[0] = chunk_size; + H5Pset_chunk(dcpl, 1, hsize); - if (compression_level > 0) { - H5Pset_deflate(dcpl, compression_level); + if (compression_level > 0) { + H5Pset_deflate(dcpl, compression_level); + } } hid_t dset = @@ -67,17 +69,18 @@ static inline bsp_error_t bsp_write_array(hid_t f, const char* label, return BSP_ERROR_IO; } - hid_t hdf5_native_type = bsp_get_hdf5_native_type(array.type); - - hid_t r = H5Dwrite(dset, hdf5_native_type, H5S_ALL, fspace, H5P_DEFAULT, - array.data); + if (array.size > 0) { + hid_t hdf5_native_type = bsp_get_hdf5_native_type(array.type); + hid_t r = H5Dwrite(dset, hdf5_native_type, H5S_ALL, fspace, H5P_DEFAULT, + array.data); - if (r < 0) { - H5Dclose(dset); - H5Sclose(fspace); - H5Pclose(lcpl); - H5Pclose(dcpl); - return BSP_ERROR_IO; + if (r < 0) { + H5Dclose(dset); + H5Sclose(fspace); + H5Pclose(lcpl); + H5Pclose(dcpl); + return BSP_ERROR_IO; + } } H5Sclose(fspace); @@ -123,12 +126,24 @@ static inline bsp_error_t bsp_read_array_parallel(bsp_array_t* array, hid_t f, hid_t hdf5_type = H5Dget_type(dset); bsp_type_t type = bsp_get_bsp_type(hdf5_type); + if (dims[0] == 0) { + array->type = type; + array->size = 0; + array->data = NULL; + array->allocator = bsp_shm_allocator; + H5Tclose(hdf5_type); + H5Sclose(fspace); + H5Dclose(dset); + return BSP_SUCCESS; + } // Array will be written into a POSIX shared memory. bsp_shm_t array_shm = bsp_shm_new(dims[0] * bsp_type_size(type)); array->type = type; array->size = dims[0]; array->allocator = bsp_shm_allocator; + array->data = bsp_shm_attach(array_shm); + bsp_shm_delete(array_shm); bsp_shm_t active_children_shm = bsp_shm_new(sizeof(_Atomic int)); @@ -152,11 +167,6 @@ static inline bsp_error_t bsp_read_array_parallel(bsp_array_t* array, hid_t f, } } - array->data = bsp_shm_attach(array_shm); - if (thread_num == 0) { - bsp_shm_delete(array_shm); - } - hsize_t chunk_size = (array->size + num_threads - 1) / num_threads; hsize_t start = (chunk_size * thread_num < array->size) ? chunk_size * thread_num @@ -237,8 +247,17 @@ static inline bsp_error_t bsp_read_array_allocator(bsp_array_t* array, hid_t f, return BSP_ERROR_MEMORY; } - herr_t status = H5Dread(dset, bsp_get_hdf5_native_type(type), H5S_ALL, - H5S_ALL, H5P_DEFAULT, array->data); + if (array->size == 0) { + H5Tclose(hdf5_type); + H5Sclose(fspace); + H5Dclose(dset); + return BSP_SUCCESS; + } + + hid_t native_type = bsp_get_hdf5_native_type(type); + herr_t status = + H5Dread(dset, native_type, H5S_ALL, H5S_ALL, H5P_DEFAULT, array->data); + H5Tclose(hdf5_type); if (status < 0) { bsp_destroy_array_t(array); diff --git a/include/binsparse/version.h b/include/binsparse/version.h new file mode 100644 index 0000000..5a47da7 --- /dev/null +++ b/include/binsparse/version.h @@ -0,0 +1,21 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include + +#define BINSPARSE_VERSION "0.1" + +#ifdef __cplusplus +extern "C" { +#endif + +bsp_error_t bsp_check_version_compatible(const char* version); + +#ifdef __cplusplus +} +#endif diff --git a/include/binsparse/write_tensor.h b/include/binsparse/write_tensor.h index 27e1f29..feef5f3 100644 --- a/include/binsparse/write_tensor.h +++ b/include/binsparse/write_tensor.h @@ -10,20 +10,19 @@ extern "C" { #endif -// TODO: make cJSON optional. - #include -#include #ifdef BSP_USE_HDF5 #include -int bsp_write_tensor_to_group(hid_t f, bsp_tensor_t tensor, cJSON* user_json, - int compression_level); +bsp_error_t bsp_write_tensor_to_group(hid_t f, bsp_tensor_t tensor, + const char* user_json, + int compression_level); #endif -int bsp_write_tensor(const char* fname, bsp_tensor_t tensor, const char* group, - cJSON* user_json, int compression_level); +bsp_error_t bsp_write_tensor(const char* fname, bsp_tensor_t tensor, + const char* group, const char* user_json, + int compression_level); #ifdef __cplusplus } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f966494..f63da43 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,6 +5,7 @@ target_sources(binsparse PRIVATE read_matrix.c read_tensor.c + version.c write_matrix.c write_tensor.c ) @@ -12,6 +13,7 @@ target_sources(binsparse PRIVATE target_sources(binsparse_dynamic PRIVATE read_matrix.c read_tensor.c + version.c write_matrix.c write_tensor.c ) diff --git a/src/read_matrix.c b/src/read_matrix.c index 1fc3877..09b96c3 100644 --- a/src/read_matrix.c +++ b/src/read_matrix.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -91,7 +92,12 @@ bsp_error_t bsp_read_matrix_from_group_parallel(bsp_matrix_t* matrix, hid_t f, assert(cJSON_IsString(version_)); - // TODO: check version. + error = bsp_check_version_compatible(cJSON_GetStringValue(version_)); + if (error != BSP_SUCCESS) { + cJSON_Delete(j); + free(json_string); + return error; + } cJSON* format_ = cJSON_GetObjectItemCaseSensitive(binsparse, "format"); assert(format_ != NULL); @@ -148,6 +154,8 @@ bsp_error_t bsp_read_matrix_from_group_parallel(bsp_matrix_t* matrix, hid_t f, // TODO: handle error return error; } + } else if (strncmp(type_string, "bint8", 5) == 0) { + matrix->values.type = BSP_BINT8; } } @@ -235,7 +243,12 @@ bsp_error_t bsp_read_matrix_from_group_allocator(bsp_matrix_t* matrix, hid_t f, return BSP_ERROR_FORMAT; } - // TODO: check version. + error = bsp_check_version_compatible(cJSON_GetStringValue(version_)); + if (error != BSP_SUCCESS) { + cJSON_Delete(j); + allocator.free(json_string); + return error; + } cJSON* format_ = cJSON_GetObjectItemCaseSensitive(binsparse, "format"); if (format_ == NULL || !cJSON_IsString(format_)) { @@ -308,6 +321,8 @@ bsp_error_t bsp_read_matrix_from_group_allocator(bsp_matrix_t* matrix, hid_t f, // TODO: handle error return error; } + } else if (strncmp(type_string, "bint8", 5) == 0) { + matrix->values.type = BSP_BINT8; } } diff --git a/src/read_tensor.c b/src/read_tensor.c index cce6004..ff1bc98 100644 --- a/src/read_tensor.c +++ b/src/read_tensor.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -52,6 +53,13 @@ bsp_tensor_t bsp_read_tensor_from_group(hid_t f) { assert(cJSON_IsString(version_)); + error = bsp_check_version_compatible(cJSON_GetStringValue(version_)); + if (error != BSP_SUCCESS) { + cJSON_Delete(j); + free(json_string); + return tensor; + } + // nnz computation cJSON* nnz_ = cJSON_GetObjectItemCaseSensitive(binsparse, "number_of_stored_values"); @@ -68,11 +76,17 @@ bsp_tensor_t bsp_read_tensor_from_group(hid_t f) { dims[idx] = cJSON_GetNumberValue(cJSON_GetArrayItem(shape_, idx)); } tensor.dims = dims; - assert(tensor.rank > 0); cJSON* data_types_ = cJSON_GetObjectItemCaseSensitive(binsparse, "data_types"); assert(data_types_ != NULL); + const char* values_type = cJSON_GetStringValue( + cJSON_GetObjectItemCaseSensitive(data_types_, "values")); + assert(values_type != NULL); + if (strncmp(values_type, "iso[", 4) == 0) { + tensor.is_iso = true; + values_type += 4; + } cJSON* binsparse_custom = cJSON_GetObjectItemCaseSensitive(binsparse, "custom"); @@ -113,6 +127,16 @@ bsp_tensor_t bsp_read_tensor_from_group(hid_t f) { free(json_string); return tensor; } + if (strncmp(values_type, "complex[", 8) == 0) { + error = bsp_fp_array_to_complex(&values); + if (error != BSP_SUCCESS) { + bsp_destroy_array_t(&values); + free(json_string); + return tensor; + } + } else if (strncmp(values_type, "bint8", 5) == 0) { + values.type = BSP_BINT8; + } cur_level->kind = BSP_TENSOR_ELEMENT; bsp_element_t* data = malloc(sizeof(bsp_element_t)); data->values = values; diff --git a/src/version.c b/src/version.c new file mode 100644 index 0000000..bea2bfa --- /dev/null +++ b/src/version.c @@ -0,0 +1,46 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +#include + +#include + +static bool parse_version(const char* text, semver_t* version) { + return text != NULL && semver_parse(text, version) == 0; +} + +static bool compatible(const semver_t* version, const semver_t* supported) { + if (version->major != supported->major) + return false; + + if (supported->major == 0) + return semver_compare_version(*version, *supported) == 0; + + return version->minor >= supported->minor; +} + +bsp_error_t bsp_check_version_compatible(const char* version) { + semver_t parsed = {0}; + semver_t supported = {0}; + bsp_error_t error = BSP_SUCCESS; + bool parsed_ok = parse_version(version, &parsed); + bool supported_ok = parsed_ok && parse_version(BINSPARSE_VERSION, &supported); + + if (!parsed_ok || !supported_ok) { + semver_free(&parsed); + semver_free(&supported); + return BSP_ERROR_FORMAT; + } + + if (!compatible(&parsed, &supported)) + error = BSP_ERROR_UNSUPPORTED; + + semver_free(&parsed); + semver_free(&supported); + return error; +} diff --git a/src/write_matrix.c b/src/write_matrix.c index fa1a26a..a7c09a1 100644 --- a/src/write_matrix.c +++ b/src/write_matrix.c @@ -69,17 +69,17 @@ char* bsp_generate_json(bsp_matrix_t matrix, cJSON* user_json) { free(type_string); } - if (matrix.indices_0.data != NULL) { + if (matrix.indices_0.type != BSP_INVALID_TYPE) { cJSON_AddStringToObject(data_types, "indices_0", bsp_get_type_string(matrix.indices_0.type)); } - if (matrix.indices_1.data != NULL) { + if (matrix.indices_1.type != BSP_INVALID_TYPE) { cJSON_AddStringToObject(data_types, "indices_1", bsp_get_type_string(matrix.indices_1.type)); } - if (matrix.pointers_to_1.data != NULL) { + if (matrix.pointers_to_1.type != BSP_INVALID_TYPE) { cJSON_AddStringToObject(data_types, "pointers_to_1", bsp_get_type_string(matrix.pointers_to_1.type)); } @@ -112,7 +112,7 @@ bsp_error_t bsp_write_matrix_to_group_cjson(hid_t f, bsp_matrix_t matrix, return error; } - if (matrix.indices_0.size > 0) { + if (matrix.indices_0.type != BSP_INVALID_TYPE) { error = bsp_write_array(f, (char*) "indices_0", matrix.indices_0, compression_level); if (error != BSP_SUCCESS) { @@ -120,7 +120,7 @@ bsp_error_t bsp_write_matrix_to_group_cjson(hid_t f, bsp_matrix_t matrix, } } - if (matrix.indices_1.size > 0) { + if (matrix.indices_1.type != BSP_INVALID_TYPE) { error = bsp_write_array(f, (char*) "indices_1", matrix.indices_1, compression_level); if (error != BSP_SUCCESS) { @@ -128,7 +128,7 @@ bsp_error_t bsp_write_matrix_to_group_cjson(hid_t f, bsp_matrix_t matrix, } } - if (matrix.pointers_to_1.size > 0) { + if (matrix.pointers_to_1.type != BSP_INVALID_TYPE) { error = bsp_write_array(f, (char*) "pointers_to_1", matrix.pointers_to_1, compression_level); if (error != BSP_SUCCESS) { diff --git a/src/write_tensor.c b/src/write_tensor.c index 1b71f17..60bb723 100644 --- a/src/write_tensor.c +++ b/src/write_tensor.c @@ -5,8 +5,10 @@ */ #include +#include #include #include +#include #include #include @@ -27,10 +29,12 @@ static cJSON* init_tensor_json(bsp_tensor_t tensor, cJSON* user_json) { cJSON_AddItemToObject(binsparse, "custom", binsparse_custom); cJSON_AddItemToObject(j, "binsparse", binsparse); - cJSON* userJsonItem; - - cJSON_ArrayForEach(userJsonItem, user_json) { - cJSON_AddItemToObject(j, userJsonItem->string, userJsonItem); + if (user_json != NULL) { + cJSON* user_json_item; + cJSON_ArrayForEach(user_json_item, user_json) { + cJSON* item_copy = cJSON_Duplicate(user_json_item, 1); + cJSON_AddItemToObject(j, user_json_item->string, item_copy); + } } cJSON_AddStringToObject(binsparse, "version", BINSPARSE_VERSION); @@ -55,8 +59,9 @@ static cJSON* init_tensor_json(bsp_tensor_t tensor, cJSON* user_json) { return j; } -bsp_error_t bsp_write_tensor_to_group(hid_t f, bsp_tensor_t tensor, - cJSON* user_json, int compression_level) { +bsp_error_t bsp_write_tensor_to_group_cjson(hid_t f, bsp_tensor_t tensor, + cJSON* user_json, + int compression_level) { // bsp_matrix_t matrix; cJSON* j = init_tensor_json(tensor, user_json); // tensor: @@ -167,13 +172,29 @@ bsp_error_t bsp_write_tensor_to_group(hid_t f, bsp_tensor_t tensor, return BSP_SUCCESS; } -bsp_error_t bsp_write_tensor(const char* fname, bsp_tensor_t tensor, - const char* group, cJSON* user_json, - int compression_level) { +bsp_error_t bsp_write_tensor_to_group(hid_t f, bsp_tensor_t tensor, + const char* user_json, + int compression_level) { + cJSON* user_json_cjson = NULL; + if (user_json != NULL) { + user_json_cjson = cJSON_Parse(user_json); + } + if (user_json_cjson == NULL) { + user_json_cjson = cJSON_CreateObject(); + } + bsp_error_t error = bsp_write_tensor_to_group_cjson( + f, tensor, user_json_cjson, compression_level); + cJSON_Delete(user_json_cjson); + return error; +} + +bsp_error_t bsp_write_tensor_cjson(const char* fname, bsp_tensor_t tensor, + const char* group, cJSON* user_json, + int compression_level) { if (group == NULL) { hid_t f = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); - bsp_error_t error = - bsp_write_tensor_to_group(f, tensor, user_json, compression_level); + bsp_error_t error = bsp_write_tensor_to_group_cjson(f, tensor, user_json, + compression_level); if (error != BSP_SUCCESS) { H5Fclose(f); return error; @@ -187,9 +208,32 @@ bsp_error_t bsp_write_tensor(const char* fname, bsp_tensor_t tensor, f = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); } hid_t g = H5Gcreate1(f, group, H5P_DEFAULT); - bsp_write_tensor_to_group(g, tensor, user_json, compression_level); + bsp_error_t error = bsp_write_tensor_to_group_cjson(g, tensor, user_json, + compression_level); + if (error != BSP_SUCCESS) { + H5Gclose(g); + H5Fclose(f); + return error; + } H5Gclose(g); H5Fclose(f); } return BSP_SUCCESS; } + +bsp_error_t bsp_write_tensor(const char* fname, bsp_tensor_t tensor, + const char* group, const char* user_json, + int compression_level) { + cJSON* user_json_cjson = NULL; + if (user_json != NULL) { + user_json_cjson = cJSON_Parse(user_json); + } + if (user_json_cjson == NULL) { + user_json_cjson = cJSON_CreateObject(); + } + + bsp_error_t error = bsp_write_tensor_cjson( + fname, tensor, group, user_json_cjson, compression_level); + cJSON_Delete(user_json_cjson); + return error; +} diff --git a/test/c/CMakeLists.txt b/test/c/CMakeLists.txt index dfa65da..f02cff8 100644 --- a/test/c/CMakeLists.txt +++ b/test/c/CMakeLists.txt @@ -5,6 +5,13 @@ add_executable(dvec_shape_test dvec_shape_test.c) target_link_libraries(dvec_shape_test binsparse) +add_executable(version_test version_test.c) +target_link_libraries(version_test binsparse) + add_test( NAME unit.dvec_shape COMMAND dvec_shape_test ${CMAKE_CURRENT_BINARY_DIR}/dvec_shape_test.h5) + +add_test( + NAME unit.version + COMMAND version_test ${CMAKE_CURRENT_BINARY_DIR}/version_test.h5) diff --git a/test/c/dvec_shape_test.c b/test/c/dvec_shape_test.c index 1e0914a..d8b1f2c 100644 --- a/test/c/dvec_shape_test.c +++ b/test/c/dvec_shape_test.c @@ -52,7 +52,7 @@ static void check_read(const char* filename, bool parallel) { static void create_invalid_shape(const char* filename) { static const char* json = - "{\"binsparse\":{\"version\":\"0.1\",\"format\":\"DVEC\"," + "{\"binsparse\":{\"version\":\"0.1.0\",\"format\":\"DVEC\"," "\"shape\":[3,1],\"number_of_stored_values\":3," "\"data_types\":{\"values\":\"float64\"}}}"; diff --git a/test/c/version_test.c b/test/c/version_test.c new file mode 100644 index 0000000..5309a0e --- /dev/null +++ b/test/c/version_test.c @@ -0,0 +1,89 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include +#include +#include + +static void write_versioned_dvec(const char* filename, const char* version) { + unlink(filename); + + bsp_matrix_t matrix; + bsp_construct_default_matrix_t(&matrix); + matrix.format = BSP_DVEC; + matrix.structure = BSP_GENERAL; + matrix.nrows = 2; + matrix.ncols = 1; + matrix.nnz = 2; + assert(bsp_construct_array_t(&matrix.values, 2, BSP_FLOAT64) == BSP_SUCCESS); + double* values = matrix.values.data; + values[0] = 1; + values[1] = 2; + + assert(bsp_write_matrix(filename, matrix, NULL, NULL, 0) == BSP_SUCCESS); + bsp_destroy_matrix_t(&matrix); + + char json[1024]; + int length = + snprintf(json, sizeof(json), + "{\"binsparse\":{\"version\":\"%s\",\"format\":\"DVEC\"," + "\"shape\":[2],\"number_of_stored_values\":2," + "\"data_types\":{\"values\":\"float64\"}}}", + version); + assert(length > 0 && (size_t) length < sizeof(json)); + + hid_t file = H5Fopen(filename, H5F_ACC_RDWR, H5P_DEFAULT); + assert(file >= 0); + H5Adelete(file, "binsparse"); + assert(bsp_write_attribute(file, "binsparse", json) == BSP_SUCCESS); + H5Fclose(file); +} + +static void check_matrix_reader(const char* filename, const char* version, + bsp_error_t expected) { + write_versioned_dvec(filename, version); + + bsp_matrix_t matrix; + bsp_error_t error = bsp_read_matrix(&matrix, filename, NULL); + assert(error == expected); + if (error == BSP_SUCCESS) { + bsp_destroy_matrix_t(&matrix); + } + + unlink(filename); +} + +int main(int argc, char** argv) { + assert(argc == 2); + const char* filename = argv[1]; + + assert(bsp_check_version_compatible("0.1") == BSP_SUCCESS); + assert(bsp_check_version_compatible("0.1.0") == BSP_SUCCESS); + assert(bsp_check_version_compatible("0.1.0+roundtrip.1") == BSP_SUCCESS); + assert(bsp_check_version_compatible("0.1.1") == BSP_ERROR_UNSUPPORTED); + assert(bsp_check_version_compatible("0.1.42+roundtrip.1") == + BSP_ERROR_UNSUPPORTED); + assert(bsp_check_version_compatible("0.1.0-alpha.1") == BSP_SUCCESS); + assert(bsp_check_version_compatible("0.2.0-alpha.1") == + BSP_ERROR_UNSUPPORTED); + assert(bsp_check_version_compatible("0.2.0") == BSP_ERROR_UNSUPPORTED); + assert(bsp_check_version_compatible("1.0.0") == BSP_ERROR_UNSUPPORTED); + assert(bsp_check_version_compatible("not-a-version") == BSP_ERROR_FORMAT); + assert(bsp_check_version_compatible(NULL) == BSP_ERROR_FORMAT); + + check_matrix_reader(filename, "0.1.0", BSP_SUCCESS); + check_matrix_reader(filename, "0.1", BSP_SUCCESS); + check_matrix_reader(filename, "0.1.0+roundtrip.1", BSP_SUCCESS); + check_matrix_reader(filename, "0.1.7", BSP_ERROR_UNSUPPORTED); + check_matrix_reader(filename, "0.1.0-alpha.1", BSP_SUCCESS); + check_matrix_reader(filename, "0.2.0-alpha.1", BSP_ERROR_UNSUPPORTED); + check_matrix_reader(filename, "0.2.0", BSP_ERROR_UNSUPPORTED); + check_matrix_reader(filename, "1.0.0", BSP_ERROR_UNSUPPORTED); + check_matrix_reader(filename, "not-a-version", BSP_ERROR_FORMAT); + + return 0; +} diff --git a/test/compliance/CMakeLists.txt b/test/compliance/CMakeLists.txt new file mode 100644 index 0000000..1cc90a7 --- /dev/null +++ b/test/compliance/CMakeLists.txt @@ -0,0 +1,25 @@ +#[[ +SPDX-FileCopyrightText: 2026 Binsparse Developers + +SPDX-License-Identifier: BSD-3-Clause +]] + +add_library(npy_bridge STATIC + npy_c.c +) +target_include_directories(npy_bridge PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} +) + +add_library(compliance_core STATIC reformat.c) +target_include_directories(compliance_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(compliance_core PUBLIC binsparse) + +add_executable(npy_to_binsparse npy_to_binsparse.c) +target_link_libraries(npy_to_binsparse PRIVATE compliance_core npy_bridge) + +add_executable(binsparse_to_npy binsparse_to_npy.c) +target_link_libraries(binsparse_to_npy PRIVATE compliance_core npy_bridge) + +add_executable(binsparse_to_binsparse binsparse_to_binsparse.c) +target_link_libraries(binsparse_to_binsparse PRIVATE compliance_core) diff --git a/test/compliance/binsparse_to_binsparse.c b/test/compliance/binsparse_to_binsparse.c new file mode 100644 index 0000000..5dde053 --- /dev/null +++ b/test/compliance/binsparse_to_binsparse.c @@ -0,0 +1,29 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "reformat.h" + +#include + +int main(int argc, char** argv) { + cJSON* header; + bsp_error_t error; + + if (argc != 3) { + fprintf(stderr, "usage: binsparse_to_binsparse \n"); + return 2; + } + header = input_header(argv[1]); + canonicalize_predefined_alias(header); + error = bsp_reformat_file(argv[1], argv[2], header, 0); + cJSON_Delete(header); + if (error != BSP_SUCCESS) { + fprintf(stderr, "binsparse_to_binsparse: %s\n", + bsp_get_error_string(error)); + return 1; + } + return 0; +} diff --git a/test/compliance/binsparse_to_npy.c b/test/compliance/binsparse_to_npy.c new file mode 100644 index 0000000..7cd2439 --- /dev/null +++ b/test/compliance/binsparse_to_npy.c @@ -0,0 +1,143 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "npy_c.h" +#include "reformat.h" + +#include +#include +#include +#include +#include + +static size_t shape_size(const size_t* shape, int rank) { + size_t size = 1; + for (int d = 0; d < rank; ++d) + size *= shape[d]; + return size; +} + +int main(int argc, char** argv) { + cJSON* header; + const char* format; + bool custom; + bsp_tensor_t tensor = bsp_construct_default_tensor_t(); + bsp_matrix_t matrix; + bsp_array_t values, fill_value; + entries_t entries; + size_t *shape, *transpose, size; + void* dense; + uint8_t* pattern; + bool owns_fill = false; + + if (argc != 5) { + fprintf(stderr, "usage: binsparse_to_npy " + " \n"); + return 2; + } + header = input_header(argv[1]); + format = + cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(header, "format")); + if (format == NULL) + die("input header has no format"); + custom = !strcmp(format, "custom"); + int rank; + if (custom) { + tensor = bsp_read_tensor(argv[1], NULL); + tensor.is_iso = header_values_are_iso(header); + rank = tensor.rank; + shape = tensor.dims; + values = bsp_get_tensor_values(tensor); + entries = tensor_to_coo(&tensor); + } else { + if (bsp_read_matrix(&matrix, argv[1], NULL) != BSP_SUCCESS) + die("cannot read Binsparse matrix"); + rank = bsp_matrix_format_is_vector(matrix.format) ? 1 : 2; + shape = malloc((size_t) rank * sizeof(size_t)); + shape[0] = matrix.nrows; + if (rank == 2) + shape[1] = matrix.ncols; + values = matrix.values; + entries = matrix_to_coo(&matrix); + } + transpose = rank ? malloc((size_t) rank * sizeof(size_t)) : NULL; + for (int d = 0; d < rank; ++d) + transpose[d] = (size_t) d; + if (custom && tensor.transpose) + memcpy(transpose, tensor.transpose, (size_t) rank * sizeof(size_t)); + else if (!custom && rank == 2 && + (matrix.format == BSP_DMATC || matrix.format == BSP_CSC || + matrix.format == BSP_DCSC || matrix.format == BSP_COOC)) { + transpose[0] = 1; + transpose[1] = 0; + } + + bsp_construct_default_array_t(&fill_value); + cJSON* fill = cJSON_GetObjectItemCaseSensitive(header, "fill"); + if (cJSON_IsTrue(fill)) { + hid_t file = H5Fopen(argv[1], H5F_ACC_RDONLY, H5P_DEFAULT); + if (file < 0 || + bsp_read_array(&fill_value, file, "fill_value") != BSP_SUCCESS) + die("cannot read fill_value"); + H5Fclose(file); + const char* fill_type = + cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive( + cJSON_GetObjectItemCaseSensitive(header, "data_types"), + "fill_value")); + if (fill_type != NULL && strncmp(fill_type, "complex[", 8) == 0 && + bsp_fp_array_to_complex(&fill_value) != BSP_SUCCESS) + die("invalid complex fill_value"); + if (fill_type != NULL && strncmp(fill_type, "bint8", 5) == 0) + fill_value.type = BSP_BINT8; + owns_fill = true; + } else { + if (bsp_construct_array_t(&fill_value, 1, values.type) != BSP_SUCCESS) + die("out of memory"); + memset(fill_value.data, 0, bsp_type_size(values.type)); + owns_fill = true; + } + size = shape_size(shape, rank); + dense = malloc(size * bsp_type_size(values.type)); + pattern = calloc(size, 1); + if ((size && !dense) || (size && !pattern)) + die("out of memory"); + for (size_t i = 0; i < size; ++i) + memcpy((char*) dense + i * bsp_type_size(values.type), fill_value.data, + bsp_type_size(values.type)); + for (size_t k = 0; k < entries.size; ++k) { + size_t linear = 0; + for (int logical = 0; logical < rank; ++logical) { + int stored = 0; + while (stored < rank && transpose[stored] != (size_t) logical) + ++stored; + linear = linear * shape[logical] + entries.entry[k].coord[stored]; + } + pattern[linear] = 1; + memcpy((char*) dense + linear * bsp_type_size(values.type), + (char*) values.data + + entries.entry[k].value * bsp_type_size(values.type), + bsp_type_size(values.type)); + } + if (npy_c_save(argv[2], dense, shape, (size_t) rank, values.type) || + npy_c_save(argv[3], pattern, shape, (size_t) rank, BSP_BINT8) || + npy_c_save(argv[4], fill_value.data, NULL, 0, fill_value.type)) + die("cannot write NPY output"); + + free(dense); + free(pattern); + free(transpose); + free_entries(&entries); + if (owns_fill) + bsp_destroy_array_t(&fill_value); + if (custom) + bsp_destroy_tensor_t(tensor); + else { + free(shape); + bsp_destroy_matrix_t(&matrix); + } + cJSON_Delete(header); + return 0; +} diff --git a/test/compliance/npy_c.c b/test/compliance/npy_c.c new file mode 100644 index 0000000..6565d2e --- /dev/null +++ b/test/compliance/npy_c.c @@ -0,0 +1,375 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "npy_c.h" + +#include +#include +#include +#include +#include +#include + +typedef struct { + size_t word_size; + bool fortran_order; + size_t rank; + size_t* shape; +} npy_header_t; + +static const char* skip_spaces(const char* text) { + while (*text == ' ' || *text == '\t' || *text == '\n' || *text == '\r') + ++text; + return text; +} + +static const char* value_for_key(const char* header, const char* key) { + const char* key_start = strstr(header, key); + if (key_start == NULL) + return NULL; + const char* value = skip_spaces(key_start + strlen(key)); + if (*value != ':') + return NULL; + return skip_spaces(value + 1); +} + +static int parse_descr(const char* header, char* kind, size_t* word_size) { + const char* value = value_for_key(header, "'descr'"); + if (value == NULL) + value = value_for_key(header, "\"descr\""); + if (value == NULL || (*value != '\'' && *value != '"')) + return 1; + + char quote = *value++; + char endian = *value++; + if (endian != '<' && endian != '>' && endian != '|' && endian != '=') + return 1; + *kind = *value++; + if (*kind != 'b' && *kind != 'i' && *kind != 'u' && *kind != 'f' && + *kind != 'c') + return 1; + + char* end = NULL; + unsigned long parsed = strtoul(value, &end, 10); + if (end == value || *end != quote || parsed == 0) + return 1; + *word_size = (size_t) parsed; + return 0; +} + +static int parse_fortran_order(const char* header, bool* fortran_order) { + const char* value = value_for_key(header, "'fortran_order'"); + if (value == NULL) + value = value_for_key(header, "\"fortran_order\""); + if (value == NULL) + return 1; + if (!strncmp(value, "True", 4)) { + *fortran_order = true; + return 0; + } + if (!strncmp(value, "False", 5)) { + *fortran_order = false; + return 0; + } + return 1; +} + +static int append_shape_dim(size_t** shape, size_t* rank, size_t dim) { + size_t* resized = realloc(*shape, (*rank + 1) * sizeof(size_t)); + if (resized == NULL) + return 1; + resized[*rank] = dim; + *shape = resized; + ++*rank; + return 0; +} + +static int parse_shape(const char* header, size_t** shape, size_t* rank) { + const char* value = value_for_key(header, "'shape'"); + if (value == NULL) + value = value_for_key(header, "\"shape\""); + if (value == NULL) + return 1; + value = skip_spaces(value); + if (*value != '(') + return 1; + ++value; + + *shape = NULL; + *rank = 0; + for (;;) { + value = skip_spaces(value); + if (*value == ')') + return 0; + + char* end = NULL; + unsigned long parsed = strtoul(value, &end, 10); + if (end == value) + return 1; + if (append_shape_dim(shape, rank, (size_t) parsed)) + return 1; + + value = skip_spaces(end); + if (*value == ',') { + ++value; + continue; + } + if (*value == ')') + return 0; + return 1; + } +} + +static void destroy_header(npy_header_t* header) { + free(header->shape); + memset(header, 0, sizeof(*header)); +} + +static uint16_t read_le16(const uint8_t bytes[2]) { + return (uint16_t) bytes[0] | ((uint16_t) bytes[1] << 8); +} + +static uint32_t read_le32(const uint8_t bytes[4]) { + return (uint32_t) bytes[0] | ((uint32_t) bytes[1] << 8) | + ((uint32_t) bytes[2] << 16) | ((uint32_t) bytes[3] << 24); +} + +static int read_header(FILE* file, npy_header_t* out) { + uint8_t prefix[10]; + if (fread(prefix, 1, sizeof(prefix), file) != sizeof(prefix)) + return 1; + const uint8_t magic[] = {0x93, 'N', 'U', 'M', 'P', 'Y'}; + if (memcmp(prefix, magic, sizeof(magic))) + return 1; + + uint8_t major = prefix[6]; + uint8_t minor = prefix[7]; + size_t header_size; + if (major == 1 && minor == 0) { + header_size = read_le16(prefix + 8); + } else if ((major == 2 || major == 3) && minor == 0) { + uint8_t length[4]; + length[0] = prefix[8]; + length[1] = prefix[9]; + if (fread(length + 2, 1, 2, file) != 2) + return 1; + header_size = read_le32(length); + } else { + return 1; + } + + char* header = malloc(header_size + 1); + if (header == NULL) + return 1; + if (fread(header, 1, header_size, file) != header_size) { + free(header); + return 1; + } + header[header_size] = '\0'; + + char kind; + int error = parse_descr(header, &kind, &out->word_size) || + parse_fortran_order(header, &out->fortran_order) || + parse_shape(header, &out->shape, &out->rank); + free(header); + return error; +} + +static int checked_mul(size_t lhs, size_t rhs, size_t* result) { + if (rhs != 0 && lhs > (size_t) -1 / rhs) + return 1; + *result = lhs * rhs; + return 0; +} + +static int element_count(const size_t* shape, size_t rank, size_t* count) { + *count = 1; + for (size_t i = 0; i < rank; ++i) + if (checked_mul(*count, shape[i], count)) + return 1; + return 0; +} + +extern int npy_c_load(const char* path, npy_c_array* out) { + FILE* file = fopen(path, "rb"); + if (file == NULL) + return 1; + + npy_header_t header = {0}; + if (read_header(file, &header)) { + fclose(file); + destroy_header(&header); + return 1; + } + + size_t count; + size_t bytes; + if (element_count(header.shape, header.rank, &count) || + checked_mul(count, header.word_size, &bytes)) { + fclose(file); + destroy_header(&header); + return 1; + } + + out->size = count; + out->word_size = header.word_size; + out->rank = header.rank; + out->shape = header.rank ? malloc(header.rank * sizeof(size_t)) : NULL; + out->strides = header.rank ? malloc(header.rank * sizeof(size_t)) : NULL; + out->data = bytes ? malloc(bytes) : NULL; + if ((header.rank && (!out->shape || !out->strides)) || + (bytes && out->data == NULL)) { + fclose(file); + destroy_header(&header); + npy_c_destroy(out); + return 1; + } + if (header.rank) + memcpy(out->shape, header.shape, header.rank * sizeof(size_t)); + + size_t stride = 1; + if (header.fortran_order) { + for (size_t d = 0; d < out->rank; ++d) { + out->strides[d] = stride; + stride *= out->shape[d]; + } + } else { + for (size_t d = out->rank; d-- > 0;) { + out->strides[d] = stride; + stride *= out->shape[d]; + } + } + + int error = bytes && fread(out->data, 1, bytes, file) != bytes; + fclose(file); + destroy_header(&header); + return error; +} + +extern void npy_c_destroy(npy_c_array* array) { + free(array->data); + free(array->shape); + free(array->strides); + memset(array, 0, sizeof(*array)); +} + +static char endian_char(void) { + const uint16_t value = 1; + return *(const uint8_t*) &value ? '<' : '>'; +} + +static int write_shape(char* buffer, size_t size, const size_t* shape, + size_t rank) { + size_t used = 0; + int written = snprintf(buffer, size, "("); + if (written < 0 || (size_t) written >= size) + return -1; + used += (size_t) written; + + for (size_t i = 0; i < rank; ++i) { + written = snprintf(buffer + used, size - used, "%s%zu", i == 0 ? "" : ", ", + shape[i]); + if (written < 0 || (size_t) written >= size - used) + return -1; + used += (size_t) written; + } + if (rank == 1) { + written = snprintf(buffer + used, size - used, ","); + if (written < 0 || (size_t) written >= size - used) + return -1; + used += (size_t) written; + } + written = snprintf(buffer + used, size - used, ")"); + if (written < 0 || (size_t) written >= size - used) + return -1; + return 0; +} + +static int write_npy(const char* path, const void* data, const size_t* shape, + size_t rank, size_t word_size, char kind) { + FILE* file = fopen(path, "wb"); + if (file == NULL) + return 1; + + char shape_text[512]; + if (write_shape(shape_text, sizeof(shape_text), shape, rank)) { + fclose(file); + return 1; + } + + char header[1024]; + char endian = word_size == 1 ? '|' : endian_char(); + int header_length = + snprintf(header, sizeof(header), + "{'descr': '%c%c%zu', 'fortran_order': False, 'shape': %s, }", + endian, kind, word_size, shape_text); + if (header_length < 0 || (size_t) header_length >= sizeof(header)) { + fclose(file); + return 1; + } + + size_t header_size = (size_t) header_length; + size_t padding = 16 - (10 + header_size) % 16; + if (header_size + padding >= sizeof(header)) { + fclose(file); + return 1; + } + memset(header + header_size, ' ', padding); + header_size += padding; + header[header_size - 1] = '\n'; + + const uint8_t magic[] = {0x93, 'N', 'U', 'M', 'P', 'Y', 0x01, 0x00}; + uint8_t header_size_bytes[] = { + (uint8_t) (header_size & 0xff), + (uint8_t) ((header_size >> 8) & 0xff), + }; + size_t count; + if (element_count(shape, rank, &count)) { + fclose(file); + return 1; + } + + int ok = fwrite(magic, 1, sizeof(magic), file) == sizeof(magic) && + fwrite(header_size_bytes, 1, sizeof(header_size_bytes), file) == + sizeof(header_size_bytes) && + fwrite(header, 1, header_size, file) == header_size && + (count == 0 || fwrite(data, word_size, count, file) == count); + return fclose(file) == 0 && ok ? 0 : 1; +} + +extern int npy_c_save(const char* path, const void* data, const size_t* shape, + size_t rank, int type) { + switch (type) { + case 0: + return write_npy(path, data, shape, rank, sizeof(uint8_t), 'u'); + case 1: + return write_npy(path, data, shape, rank, sizeof(uint16_t), 'u'); + case 2: + return write_npy(path, data, shape, rank, sizeof(uint32_t), 'u'); + case 3: + return write_npy(path, data, shape, rank, sizeof(uint64_t), 'u'); + case 4: + return write_npy(path, data, shape, rank, sizeof(int8_t), 'i'); + case 5: + return write_npy(path, data, shape, rank, sizeof(int16_t), 'i'); + case 6: + return write_npy(path, data, shape, rank, sizeof(int32_t), 'i'); + case 7: + return write_npy(path, data, shape, rank, sizeof(int64_t), 'i'); + case 8: + return write_npy(path, data, shape, rank, sizeof(float), 'f'); + case 9: + return write_npy(path, data, shape, rank, sizeof(double), 'f'); + case 10: + return write_npy(path, data, shape, rank, sizeof(bool), 'b'); + case 11: + return write_npy(path, data, shape, rank, sizeof(float _Complex), 'c'); + case 12: + return write_npy(path, data, shape, rank, sizeof(double _Complex), 'c'); + default: + return 1; + } +} diff --git a/test/compliance/npy_c.h b/test/compliance/npy_c.h new file mode 100644 index 0000000..930278b --- /dev/null +++ b/test/compliance/npy_c.h @@ -0,0 +1,31 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + void* data; + size_t size; + size_t word_size; + size_t rank; + size_t* shape; + size_t* strides; +} npy_c_array; + +int npy_c_load(const char* path, npy_c_array* array); +void npy_c_destroy(npy_c_array* array); +int npy_c_save(const char* path, const void* data, const size_t* shape, + size_t rank, int bsp_type); + +#ifdef __cplusplus +} +#endif diff --git a/test/compliance/npy_to_binsparse.c b/test/compliance/npy_to_binsparse.c new file mode 100644 index 0000000..db48390 --- /dev/null +++ b/test/compliance/npy_to_binsparse.c @@ -0,0 +1,188 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "npy_c.h" +#include "reformat.h" + +#include + +#include +#include +#include +#include + +static size_t product(const size_t* shape, size_t rank) { + size_t size = 1; + for (size_t d = 0; d < rank; ++d) + size *= shape[d]; + return size; +} + +static size_t offset(const npy_c_array* array, const size_t* coord) { + size_t result = 0; + for (size_t d = 0; d < array->rank; ++d) + result += coord[d] * array->strides[d]; + return result; +} + +int main(int argc, char** argv) { + npy_c_array dense = {0}, pattern = {0}, fill = {0}; + cJSON *header, *shape_json, *custom, *description; + const char *format, *value_name; + bsp_type_t value_type; + bsp_array_t values; + bsp_tensor_t result = bsp_construct_default_tensor_t(); + entries_t entries = {0}; + reformat_context_t context; + size_t *logical_coord, *source_transpose, *target_transpose, *stored_dims, + root_ptr[2]; + size_t stored = 0; + bool iso; + + if (argc != 6) { + fprintf(stderr, "usage: npy_to_binsparse " + " \n"); + return 2; + } + if (npy_c_load(argv[1], &dense) || npy_c_load(argv[2], &pattern) || + npy_c_load(argv[3], &fill)) + die("cannot read NPY input"); + if (dense.rank != pattern.rank || dense.size != pattern.size || + pattern.word_size != 1 || fill.size != 1) + die("incompatible tensor, pattern, or fill-value NPY input"); + + header = read_json(argv[4]); + shape_json = cJSON_GetObjectItemCaseSensitive(header, "shape"); + result.rank = cJSON_GetArraySize(shape_json); + if ((size_t) result.rank != dense.rank) + die("header shape rank mismatch"); + result.dims = + result.rank ? malloc((size_t) result.rank * sizeof(size_t)) : NULL; + result.transpose = + result.rank ? malloc((size_t) result.rank * sizeof(size_t)) : NULL; + logical_coord = + result.rank ? calloc((size_t) result.rank, sizeof(size_t)) : NULL; + target_transpose = + result.rank ? malloc((size_t) result.rank * sizeof(size_t)) : NULL; + source_transpose = + result.rank ? malloc((size_t) result.rank * sizeof(size_t)) : NULL; + stored_dims = + result.rank ? malloc((size_t) result.rank * sizeof(size_t)) : NULL; + entries.rank = result.rank; + for (int d = 0; d < result.rank; ++d) { + result.dims[d] = + (size_t) cJSON_GetNumberValue(cJSON_GetArrayItem(shape_json, d)); + if (result.dims[d] != dense.shape[d]) + die("header shape mismatch"); + source_transpose[d] = result.transpose[d] = target_transpose[d] = + (size_t) d; + } + if (product(result.dims, (size_t) result.rank) != dense.size) + die("NPY element count does not match shape"); + + context.data_types = cJSON_GetObjectItemCaseSensitive(header, "data_types"); + value_name = cJSON_GetStringValue( + cJSON_GetObjectItemCaseSensitive(context.data_types, "values")); + value_type = type_from_name(value_name); + if (value_type == BSP_INVALID_TYPE || + bsp_type_size(value_type) != dense.word_size) + die("NPY dtype does not match header values type"); + if (fill.word_size != bsp_type_size(value_type)) + die("fill-value dtype does not match header values type"); + if (!cJSON_HasObjectItem(header, "fill")) + cJSON_AddBoolToObject(header, "fill", true); + if (!cJSON_HasObjectItem(context.data_types, "fill_value")) + cJSON_AddStringToObject(context.data_types, "fill_value", + bsp_get_type_string(value_type)); + iso = value_name != NULL && !strncmp(value_name, "iso[", 4); + for (size_t i = 0; i < pattern.size; ++i) + if (((uint8_t*) pattern.data)[i]) + ++stored; + if (bsp_construct_array_t(&values, iso && stored ? 1 : stored, value_type) != + BSP_SUCCESS) + die("out of memory"); + + size_t value_index = 0; + for (size_t linear = 0; linear < dense.size; ++linear) { + size_t dense_offset = offset(&dense, logical_coord); + size_t pattern_offset = offset(&pattern, logical_coord); + if (((uint8_t*) pattern.data)[pattern_offset]) { + push_entry(&entries, logical_coord, iso ? 0 : value_index); + if (!iso || value_index == 0) + memcpy((char*) values.data + (iso ? 0 : value_index) * dense.word_size, + (char*) dense.data + dense_offset * dense.word_size, + dense.word_size); + ++value_index; + } + for (int d = result.rank - 1; d >= 0; --d) { + if (++logical_coord[d] < result.dims[d]) + break; + logical_coord[d] = 0; + } + } + result.nnz = stored; + result.is_iso = iso; + + format = + cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(header, "format")); + if (format == NULL) + die("header has no format"); + if (!strcmp(format, "custom")) { + custom = cJSON_GetObjectItemCaseSensitive(header, "custom"); + description = + cJSON_Duplicate(cJSON_GetObjectItemCaseSensitive(custom, "level"), 1); + if (description == NULL) + die("custom target requires custom.level"); + cJSON* transpose = cJSON_GetObjectItemCaseSensitive(custom, "transpose"); + if (transpose != NULL) + for (int d = 0; d < result.rank; ++d) + result.transpose[d] = target_transpose[d] = + (size_t) cJSON_GetNumberValue(cJSON_GetArrayItem(transpose, d)); + } else { + description = predefined_level(format, target_transpose, result.rank); + if (description == NULL) + die("unknown or rank-incompatible target format"); + for (int d = 0; d < result.rank; ++d) + result.transpose[d] = target_transpose[d]; + } + apply_transpose(&entries, source_transpose, target_transpose); + bsp_array_t ordered_values = values_in_entry_order(values, &entries, iso); + bsp_destroy_array_t(&values); + values = ordered_values; + for (int d = 0; d < result.rank; ++d) + stored_dims[d] = result.dims[target_transpose[d]]; + root_ptr[0] = 0; + root_ptr[1] = stored; + result.level = build_level(&context, description, &entries, stored_dims, 0, + root_ptr, 1, true, values); + if (!strcmp(format, "custom")) { + cJSON* user = cJSON_CreateObject(); + if (bsp_write_tensor_cjson(argv[5], result, NULL, user, 0) != BSP_SUCCESS) + die("cannot write Binsparse tensor"); + cJSON_Delete(user); + } else if (write_predefined(argv[5], result, format, 0) != BSP_SUCCESS) { + die("cannot write Binsparse tensor"); + } + bsp_array_t fill_array; + fill_array.data = fill.data; + fill_array.size = 1; + fill_array.type = value_type; + fill_array.allocator = bsp_default_allocator; + complete_output(argv[5], header, &fill_array); + + bsp_destroy_tensor_t(result); + free_entries(&entries); + free(logical_coord); + free(source_transpose); + free(target_transpose); + free(stored_dims); + cJSON_Delete(description); + cJSON_Delete(header); + npy_c_destroy(&dense); + npy_c_destroy(&pattern); + npy_c_destroy(&fill); + return 0; +} diff --git a/test/compliance/reformat.c b/test/compliance/reformat.c new file mode 100644 index 0000000..ca0bd30 --- /dev/null +++ b/test/compliance/reformat.c @@ -0,0 +1,811 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * SPDX-License-Identifier: BSD-3-Clause + * + * Reformat a Binsparse matrix or tensor through a coordinate list. + * + * Usage: reformat INPUT OUTPUT TARGET_HEADER.json [COMPRESSION] + * + * INPUT may use a predefined matrix/vector alias or the custom tensor format. + * TARGET_HEADER is a Binsparse header (the object containing "format", not the + * outer {"binsparse": ...} wrapper). Both predefined aliases and arbitrary + * custom dense/sparse level trees are accepted. + */ + +#include "reformat.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +bsp_type_t type_from_name(const char* name) { + size_t length; + char base[32]; + if (name == NULL) + return BSP_INVALID_TYPE; + length = strlen(name); + if (length > 5 && !strncmp(name, "iso[", 4) && name[length - 1] == ']') { + if (length - 5 >= sizeof(base)) + return BSP_INVALID_TYPE; + memcpy(base, name + 4, length - 5); + base[length - 5] = '\0'; + name = base; + } + for (int type = BSP_UINT8; type < BSP_INVALID_TYPE; ++type) + if (!strcmp(name, bsp_get_type_string((bsp_type_t) type))) + return (bsp_type_t) type; + return BSP_INVALID_TYPE; +} + +static bsp_type_t requested_type(const reformat_context_t* context, + const char* key, bsp_type_t fallback) { + const char* name = cJSON_GetStringValue( + cJSON_GetObjectItemCaseSensitive(context->data_types, key)); + bsp_type_t type = type_from_name(name); + return type == BSP_INVALID_TYPE ? fallback : type; +} + +void die(const char* message) { + fprintf(stderr, "reformat: %s\n", message); + exit(EXIT_FAILURE); +} + +cJSON* read_json(const char* path) { + FILE* file = fopen(path, "rb"); + long size = 0; + char* text; + cJSON* json; + if (file == NULL) + die(strerror(errno)); + if (fseek(file, 0, SEEK_END) != 0 || (size = ftell(file)) < 0 || + fseek(file, 0, SEEK_SET) != 0) + die("cannot read target header"); + text = malloc((size_t) size + 1); + if (text == NULL) + die("out of memory"); + if (fread(text, 1, (size_t) size, file) != (size_t) size) + die("cannot read target header"); + fclose(file); + text[size] = '\0'; + json = cJSON_Parse(text); + free(text); + if (!cJSON_IsObject(json)) + die("target header must be a JSON object"); + return json; +} + +cJSON* input_header(const char* path) { + hid_t file = H5Fopen(path, H5F_ACC_RDONLY, H5P_DEFAULT); + char* text = NULL; + cJSON *outer, *header; + if (file < 0 || bsp_read_attribute(&text, file, "binsparse") != BSP_SUCCESS) + die("cannot read input Binsparse header"); + H5Fclose(file); + outer = cJSON_Parse(text); + free(text); + if (!cJSON_IsObject(outer)) + die("invalid input Binsparse header"); + header = cJSON_DetachItemFromObjectCaseSensitive(outer, "binsparse"); + cJSON_Delete(outer); + if (!cJSON_IsObject(header)) + die("input has no binsparse header"); + return header; +} + +bool header_values_are_iso(cJSON* header) { + cJSON* types = cJSON_GetObjectItemCaseSensitive(header, "data_types"); + const char* value = + cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(types, "values")); + return value != NULL && strncmp(value, "iso[", 4) == 0; +} + +bool header_has_fill(cJSON* header) { + return cJSON_IsTrue(cJSON_GetObjectItemCaseSensitive(header, "fill")); +} + +void canonicalize_predefined_alias(cJSON* header) { + cJSON* item = cJSON_GetObjectItemCaseSensitive(header, "format"); + const char* format = cJSON_GetStringValue(item); + const char* canonical = NULL; + if (format == NULL) + return; + if (!strcmp(format, "DMAT")) + canonical = "DMATR"; + else if (!strcmp(format, "COO")) + canonical = "COOR"; + if (canonical != NULL) + cJSON_ReplaceItemInObjectCaseSensitive(header, "format", + cJSON_CreateString(canonical)); +} + +static bsp_error_t read_fill_value(const char* path, bsp_array_t* value) { + hid_t file = H5Fopen(path, H5F_ACC_RDONLY, H5P_DEFAULT); + bsp_error_t error; + if (file < 0) + return BSP_ERROR_IO; + error = bsp_read_array(value, file, "fill_value"); + H5Fclose(file); + return error; +} + +/* Preserve every supplied partial-header field and add its fill-value buffer. + */ +void complete_output(const char* path, cJSON* requested, + const bsp_array_t* fill_value) { + hid_t file = H5Fopen(path, H5F_ACC_RDWR, H5P_DEFAULT); + char *text = NULL, *updated; + cJSON *outer, *header, *item; + if (file < 0 || bsp_read_attribute(&text, file, "binsparse") != BSP_SUCCESS) + die("cannot reopen output header"); + outer = cJSON_Parse(text); + free(text); + header = cJSON_GetObjectItemCaseSensitive(outer, "binsparse"); + cJSON_ArrayForEach(item, requested) { + cJSON* copy = cJSON_Duplicate(item, 1); + if (cJSON_HasObjectItem(header, item->string)) + cJSON_ReplaceItemInObjectCaseSensitive(header, item->string, copy); + else + cJSON_AddItemToObject(header, item->string, copy); + } + if (fill_value != NULL && + bsp_write_array(file, "fill_value", *fill_value, 0) != BSP_SUCCESS) + die("cannot write fill_value"); + updated = cJSON_Print(outer); + H5Adelete(file, "binsparse"); + if (updated == NULL || + bsp_write_attribute(file, "binsparse", updated) != BSP_SUCCESS) + die("cannot update output header"); + free(updated); + cJSON_Delete(outer); + H5Fclose(file); +} + +void push_entry(entries_t* entries, const size_t* coord, size_t value) { + entry_t* item; + if (entries->size == entries->capacity) { + size_t capacity = entries->capacity == 0 ? 16 : 2 * entries->capacity; + entry_t* grown = realloc(entries->entry, capacity * sizeof(*grown)); + if (grown == NULL) + die("out of memory"); + entries->entry = grown; + entries->capacity = capacity; + } + item = &entries->entry[entries->size++]; + item->coord = malloc((entries->rank ? (size_t) entries->rank : 1) * + sizeof(*item->coord)); + if (item->coord == NULL) + die("out of memory"); + memcpy(item->coord, coord, (size_t) entries->rank * sizeof(*coord)); + item->value = value; +} + +void free_entries(entries_t* entries) { + for (size_t i = 0; i < entries->size; ++i) + free(entries->entry[i].coord); + free(entries->entry); +} + +/* Expand one custom level. begin/end identify the parent positions entering + * the level; coordinates are stored row-wise in coords. */ +static size_t stored_dimension(const bsp_tensor_t* tensor, size_t depth) { + size_t logical = tensor->transpose ? tensor->transpose[depth] : depth; + if (logical >= (size_t) tensor->rank) + die("transpose is not a permutation"); + return tensor->dims[logical]; +} + +static void level_to_coo(const bsp_level_t* level, const bsp_tensor_t* tensor, + size_t depth, size_t parents, const size_t* coords, + entries_t* out) { + if (level->kind == BSP_TENSOR_ELEMENT) { + const bsp_element_t* element = level->data; + if (parents != tensor->nnz) + die("element positions do not match number_of_stored_values"); + for (size_t p = 0; p < parents; ++p) + push_entry(out, coords + p * (size_t) tensor->rank, + tensor->is_iso ? 0 : p); + (void) element; + return; + } + + if (level->kind == BSP_TENSOR_DENSE) { + const bsp_dense_t* dense = level->data; + size_t width = 1, count, *next; + for (int d = 0; d < dense->rank; ++d) + width *= stored_dimension(tensor, depth + (size_t) d); + count = parents * width; + next = calloc(count * (size_t) tensor->rank, sizeof(*next)); + if (next == NULL && count != 0) + die("out of memory"); + for (size_t p = 0; p < parents; ++p) { + for (size_t suffix = 0; suffix < width; ++suffix) { + size_t q = p * width + suffix, n = suffix; + memcpy(next + q * (size_t) tensor->rank, + coords + p * (size_t) tensor->rank, depth * sizeof(*next)); + for (int d = dense->rank - 1; d >= 0; --d) { + size_t dim = stored_dimension(tensor, depth + (size_t) d); + next[q * (size_t) tensor->rank + depth + (size_t) d] = n % dim; + n /= dim; + } + } + } + level_to_coo(dense->child, tensor, depth + (size_t) dense->rank, count, + next, out); + free(next); + return; + } + + if (level->kind == BSP_TENSOR_SPARSE) { + const bsp_sparse_t* sparse = level->data; + size_t count = sparse->indices[0].size, *next; + size_t q = 0; + next = calloc(count * (size_t) tensor->rank, sizeof(*next)); + if (next == NULL && count != 0) + die("out of memory"); + for (size_t p = 0; p < parents; ++p) { + size_t begin = 0, end = count; + if (sparse->pointers_to != NULL) { + bsp_array_t pointers = *sparse->pointers_to; + bsp_array_read(pointers, p, begin); + bsp_array_read(pointers, p + 1, end); + } + for (size_t i = begin; i < end; ++i, ++q) { + memcpy(next + q * (size_t) tensor->rank, + coords + p * (size_t) tensor->rank, depth * sizeof(*next)); + for (int d = 0; d < sparse->rank; ++d) { + size_t index = 0; + bsp_array_read(sparse->indices[d], i, index); + if (index >= stored_dimension(tensor, depth + (size_t) d)) + die("input coordinate is out of bounds"); + next[q * (size_t) tensor->rank + depth + (size_t) d] = index; + } + } + } + if (q != count) + die("invalid sparse pointers"); + level_to_coo(sparse->child, tensor, depth + (size_t) sparse->rank, count, + next, out); + free(next); + return; + } + die("unknown tensor level"); +} + +entries_t tensor_to_coo(const bsp_tensor_t* tensor) { + entries_t result = {0}; + size_t* root; + result.rank = tensor->rank; + root = calloc(tensor->rank ? (size_t) tensor->rank : 1, sizeof(*root)); + if (root == NULL) + die("out of memory"); + level_to_coo(tensor->level, tensor, 0, 1, root, &result); + free(root); + return result; +} + +entries_t matrix_to_coo(bsp_matrix_t* matrix) { + entries_t result = {0}; + result.rank = bsp_matrix_format_is_vector(matrix->format) ? 1 : 2; + if (matrix->format == BSP_COOR || matrix->format == BSP_COOC || + matrix->format == BSP_CVEC) { + for (size_t i = 0; i < matrix->nnz; ++i) { + size_t coord[2] = {0, 0}; + bsp_array_read(matrix->indices_0, i, coord[0]); + if (result.rank == 2) + bsp_array_read(matrix->indices_1, i, coord[1]); + push_entry(&result, coord, matrix->is_iso ? 0 : i); + } + return result; + } + if (matrix->format == BSP_CSR || matrix->format == BSP_CSC) { + size_t fibers = matrix->format == BSP_CSR ? matrix->nrows : matrix->ncols; + for (size_t p = 0; p < fibers; ++p) { + size_t begin = 0, end = 0; + bsp_array_read(matrix->pointers_to_1, p, begin); + bsp_array_read(matrix->pointers_to_1, p + 1, end); + for (size_t i = begin; i < end; ++i) { + size_t coord[2] = {p, 0}; + bsp_array_read(matrix->indices_1, i, coord[1]); + push_entry(&result, coord, matrix->is_iso ? 0 : i); + } + } + return result; + } + if (matrix->format == BSP_DCSR || matrix->format == BSP_DCSC) { + for (size_t p = 0; p < matrix->indices_0.size; ++p) { + size_t begin = 0, end = 0, outer = 0; + bsp_array_read(matrix->indices_0, p, outer); + bsp_array_read(matrix->pointers_to_1, p, begin); + bsp_array_read(matrix->pointers_to_1, p + 1, end); + for (size_t i = begin; i < end; ++i) { + size_t coord[2] = {outer, 0}; + bsp_array_read(matrix->indices_1, i, coord[1]); + push_entry(&result, coord, matrix->is_iso ? 0 : i); + } + } + return result; + } + if (matrix->format == BSP_DVEC || matrix->format == BSP_DMAT || + matrix->format == BSP_DMATC) { + size_t rows = matrix->format == BSP_DMATC ? matrix->ncols : matrix->nrows; + size_t cols = + result.rank == 1 + ? 1 + : (matrix->format == BSP_DMATC ? matrix->nrows : matrix->ncols); + for (size_t i = 0; i < matrix->nnz; ++i) { + size_t coord[2] = {i / cols, i % cols}; + if (result.rank == 1) + coord[0] = i; + if (coord[0] >= rows) + die("dense value count exceeds shape"); + push_entry(&result, coord, matrix->is_iso ? 0 : i); + } + return result; + } + die("unsupported predefined input format"); + return result; +} + +/* A predefined format is only a named view of a custom level tree. */ +bsp_error_t write_predefined(const char* path, bsp_tensor_t tensor, + const char* format, int compression) { + bsp_matrix_t matrix; + bsp_level_t* level = tensor.level; + bsp_construct_default_matrix_t(&matrix); + matrix.format = bsp_get_matrix_format((char*) format); + matrix.nrows = tensor.dims[0]; + matrix.ncols = tensor.rank == 1 ? 1 : tensor.dims[1]; + matrix.nnz = tensor.nnz; + matrix.is_iso = tensor.is_iso; + matrix.structure = tensor.structure; + matrix.values = bsp_get_tensor_values(tensor); + + if (matrix.format == BSP_INVALID_FORMAT) + return BSP_ERROR_UNSUPPORTED; + if (matrix.format == BSP_DVEC || matrix.format == BSP_DMAT || + matrix.format == BSP_DMATC) { + return bsp_write_matrix(path, matrix, NULL, NULL, compression); + } + if (matrix.format == BSP_CVEC || matrix.format == BSP_COOR || + matrix.format == BSP_COOC) { + bsp_sparse_t* sparse; + if (level->kind != BSP_TENSOR_SPARSE) + return BSP_ERROR_INTERNAL; + sparse = level->data; + matrix.indices_0 = sparse->indices[0]; + if (sparse->rank == 2) + matrix.indices_1 = sparse->indices[1]; + return bsp_write_matrix(path, matrix, NULL, NULL, compression); + } + if (matrix.format == BSP_CSR || matrix.format == BSP_CSC) { + bsp_dense_t* dense; + bsp_sparse_t* sparse; + if (level->kind != BSP_TENSOR_DENSE) + return BSP_ERROR_INTERNAL; + dense = level->data; + if (dense->child->kind != BSP_TENSOR_SPARSE) + return BSP_ERROR_INTERNAL; + sparse = dense->child->data; + matrix.pointers_to_1 = *sparse->pointers_to; + matrix.indices_1 = sparse->indices[0]; + return bsp_write_matrix(path, matrix, NULL, NULL, compression); + } + if (matrix.format == BSP_DCSR || matrix.format == BSP_DCSC) { + bsp_sparse_t *outer, *inner; + if (level->kind != BSP_TENSOR_SPARSE) + return BSP_ERROR_INTERNAL; + outer = level->data; + if (outer->child->kind != BSP_TENSOR_SPARSE) + return BSP_ERROR_INTERNAL; + inner = outer->child->data; + matrix.indices_0 = outer->indices[0]; + matrix.pointers_to_1 = *inner->pointers_to; + matrix.indices_1 = inner->indices[0]; + return bsp_write_matrix(path, matrix, NULL, NULL, compression); + } + return BSP_ERROR_UNSUPPORTED; +} + +static int compare_rank; +static int compare_entry(const void* a_, const void* b_) { + const entry_t *a = a_, *b = b_; + for (int d = 0; d < compare_rank; ++d) { + if (a->coord[d] < b->coord[d]) + return -1; + if (a->coord[d] > b->coord[d]) + return 1; + } + return 0; +} + +void apply_transpose(entries_t* entries, const size_t* source, + const size_t* target) { + size_t* copy = + malloc((entries->rank ? (size_t) entries->rank : 1) * sizeof(*copy)); + if (copy == NULL) + die("out of memory"); + for (size_t k = 0; k < entries->size; ++k) { + memcpy(copy, entries->entry[k].coord, + (size_t) entries->rank * sizeof(*copy)); + for (int td = 0; td < entries->rank; ++td) { + int sd = 0; + while (sd < entries->rank && source[sd] != target[td]) + ++sd; + if (sd == entries->rank) + die("transpose is not a permutation"); + entries->entry[k].coord[td] = copy[sd]; + } + } + free(copy); + compare_rank = entries->rank; + qsort(entries->entry, entries->size, sizeof(*entries->entry), compare_entry); +} + +bsp_array_t values_in_entry_order(const bsp_array_t source, + const entries_t* entries, bool iso) { + bsp_array_t values; + size_t size = iso && entries->size != 0 ? 1 : entries->size; + if (bsp_construct_array_t(&values, size, source.type) != BSP_SUCCESS) + die("out of memory"); + for (size_t i = 0; i < size; ++i) + memcpy((char*) values.data + i * bsp_type_size(values.type), + (char*) source.data + + entries->entry[iso ? 0 : i].value * bsp_type_size(source.type), + bsp_type_size(source.type)); + return values; +} + +static size_t prefix_equal(const entry_t* a, const entry_t* b, size_t depth, + int rank) { + for (int d = 0; d < rank; ++d) + if (a->coord[depth + (size_t) d] != b->coord[depth + (size_t) d]) + return 0; + return 1; +} + +/* Recursively build a level. parent_ptr partitions entries into incoming + * fibers. The entries are already lexicographically sorted. */ +bsp_level_t* build_level(const reformat_context_t* context, cJSON* description, + const entries_t* entries, const size_t* dims, + size_t depth, const size_t* parent_ptr, size_t parents, + bool root, bsp_array_t values) { + cJSON* desc = cJSON_GetObjectItemCaseSensitive(description, "level_desc"); + bsp_level_t* level = calloc(1, sizeof(*level)); + const char* name = cJSON_GetStringValue(desc); + if (level == NULL) + die("out of memory"); + if (name != NULL && strcmp(name, "element") == 0) { + bsp_element_t* element = malloc(sizeof(*element)); + for (size_t p = 0; p < parents; ++p) + if (parent_ptr[p + 1] - parent_ptr[p] != 1) + die("each element position must have one value"); + if (element == NULL) + die("out of memory"); + element->values = values; + level->kind = BSP_TENSOR_ELEMENT; + level->data = element; + return level; + } + cJSON* rank_json = cJSON_GetObjectItemCaseSensitive(description, "rank"); + int rank = (int) cJSON_GetNumberValue(rank_json); + cJSON* child_desc = cJSON_GetObjectItemCaseSensitive(description, "level"); + if (rank <= 0 || depth + (size_t) rank > (size_t) entries->rank || + !cJSON_IsObject(child_desc)) + die("invalid target level descriptor"); + + if (strcmp(name ? name : "", "sparse") == 0) { + bsp_sparse_t* sparse = calloc(1, sizeof(*sparse)); + size_t groups = 0, *child_ptr; + if (sparse == NULL) + die("out of memory"); + for (size_t p = 0; p < parents; ++p) + for (size_t i = parent_ptr[p]; i < parent_ptr[p + 1]; ++i) + if (i == parent_ptr[p] || + !prefix_equal(&entries->entry[i - 1], &entries->entry[i], depth, + rank)) + ++groups; + sparse->rank = rank; + sparse->indices = calloc((size_t) rank, sizeof(*sparse->indices)); + child_ptr = malloc((groups + 1) * sizeof(*child_ptr)); + if (sparse->indices == NULL || child_ptr == NULL) + die("out of memory"); + for (int d = 0; d < rank; ++d) { + char key[64]; + snprintf(key, sizeof(key), "indices_%zu", depth + (size_t) d); + if (bsp_construct_array_t( + &sparse->indices[d], groups, + requested_type(context, key, + bsp_pick_integer_type( + dims[depth + (size_t) d]))) != BSP_SUCCESS) + die("out of memory"); + } + if (!root) { + char key[64]; + snprintf(key, sizeof(key), "pointers_to_%zu", depth); + sparse->pointers_to = malloc(sizeof(*sparse->pointers_to)); + if (sparse->pointers_to == NULL || + bsp_construct_array_t( + sparse->pointers_to, parents + 1, + requested_type(context, key, bsp_pick_integer_type(groups))) != + BSP_SUCCESS) + die("out of memory"); + } + groups = 0; + child_ptr[0] = 0; + for (size_t p = 0; p < parents; ++p) { + if (!root) { + bsp_array_t pointers = *sparse->pointers_to; + bsp_array_write(pointers, p, groups); + } + for (size_t i = parent_ptr[p]; i < parent_ptr[p + 1]; ++i) { + if (i == parent_ptr[p] || + !prefix_equal(&entries->entry[i - 1], &entries->entry[i], depth, + rank)) { + if (groups != 0) + child_ptr[groups] = i; + for (int d = 0; d < rank; ++d) + bsp_array_write(sparse->indices[d], groups, + entries->entry[i].coord[depth + (size_t) d]); + ++groups; + } + } + } + child_ptr[groups] = entries->size; + if (!root) { + bsp_array_t pointers = *sparse->pointers_to; + bsp_array_write(pointers, parents, groups); + } + sparse->child = + build_level(context, child_desc, entries, dims, depth + (size_t) rank, + child_ptr, groups, false, values); + free(child_ptr); + level->kind = BSP_TENSOR_SPARSE; + level->data = sparse; + return level; + } + + if (strcmp(name ? name : "", "dense") == 0) { + bsp_dense_t* dense = malloc(sizeof(*dense)); + size_t width = 1, children, *child_ptr; + if (dense == NULL) + die("out of memory"); + for (int d = 0; d < rank; ++d) + width *= dims[depth + (size_t) d]; + children = parents * width; + child_ptr = malloc((children + 1) * sizeof(*child_ptr)); + if (child_ptr == NULL) + die("out of memory"); + size_t at = 0; + for (size_t p = 0; p < parents; ++p) { + size_t end = parent_ptr[p + 1]; + for (size_t suffix = 0; suffix < width; ++suffix) { + child_ptr[p * width + suffix] = at; + while (at < end) { + size_t n = suffix; + int equal = 1; + for (int d = rank - 1; d >= 0; --d) { + size_t dim = dims[depth + (size_t) d]; + if (entries->entry[at].coord[depth + (size_t) d] != n % dim) + equal = 0; + n /= dim; + } + if (!equal) + break; + ++at; + } + } + } + child_ptr[children] = entries->size; + dense->rank = rank; + dense->child = + build_level(context, child_desc, entries, dims, depth + (size_t) rank, + child_ptr, children, false, values); + free(child_ptr); + level->kind = BSP_TENSOR_DENSE; + level->data = dense; + return level; + } + die("unknown target level descriptor"); + return NULL; +} + +cJSON* predefined_level(const char* format, size_t* transpose, int rank) { + const char* json = NULL; + if (!strcmp(format, "DVEC")) + json = "{\"level_desc\":\"dense\",\"rank\":1,\"level\":{\"level_desc\":" + "\"element\"}}"; + else if (!strcmp(format, "CVEC")) + json = "{\"level_desc\":\"sparse\",\"rank\":1,\"level\":{\"level_desc\":" + "\"element\"}}"; + else if (!strcmp(format, "DMAT") || !strcmp(format, "DMATR")) + json = "{\"level_desc\":\"dense\",\"rank\":2,\"level\":{\"level_desc\":" + "\"element\"}}"; + else if (!strcmp(format, "DMATC")) { + json = "{\"level_desc\":\"dense\",\"rank\":2,\"level\":{\"level_desc\":" + "\"element\"}}"; + transpose[0] = 1; + transpose[1] = 0; + } else if (!strcmp(format, "COO") || !strcmp(format, "COOR")) + json = "{\"level_desc\":\"sparse\",\"rank\":2,\"level\":{\"level_desc\":" + "\"element\"}}"; + else if (!strcmp(format, "COOC")) { + json = "{\"level_desc\":\"sparse\",\"rank\":2,\"level\":{\"level_desc\":" + "\"element\"}}"; + transpose[0] = 1; + transpose[1] = 0; + } else if (!strcmp(format, "CSR")) + json = "{\"level_desc\":\"dense\",\"rank\":1,\"level\":{\"level_desc\":" + "\"sparse\",\"rank\":1,\"level\":{\"level_desc\":\"element\"}}}"; + else if (!strcmp(format, "CSC")) { + json = "{\"level_desc\":\"dense\",\"rank\":1,\"level\":{\"level_desc\":" + "\"sparse\",\"rank\":1,\"level\":{\"level_desc\":\"element\"}}}"; + transpose[0] = 1; + transpose[1] = 0; + } else if (!strcmp(format, "DCSR")) + json = "{\"level_desc\":\"sparse\",\"rank\":1,\"level\":{\"level_desc\":" + "\"sparse\",\"rank\":1,\"level\":{\"level_desc\":\"element\"}}}"; + else if (!strcmp(format, "DCSC")) { + json = "{\"level_desc\":\"sparse\",\"rank\":1,\"level\":{\"level_desc\":" + "\"sparse\",\"rank\":1,\"level\":{\"level_desc\":\"element\"}}}"; + transpose[0] = 1; + transpose[1] = 0; + } + if (json == NULL || (rank != 1 && rank != 2)) + return NULL; + return cJSON_Parse(json); +} + +bsp_error_t bsp_reformat_file(const char* input, const char* output, + cJSON* target_header, int compression) { + cJSON *source_header, *format_json, *custom, *description; + reformat_context_t context; + const char *source_format, *target_format; + bsp_tensor_t tensor = bsp_construct_default_tensor_t(); + bsp_matrix_t matrix; + bsp_array_t source_values, values; + entries_t entries; + size_t *source_transpose, *target_transpose, *stored_dims, root_ptr[2]; + int rank; + bool source_is_tensor, iso; + bsp_array_t fill_value; + bool owns_fill = false; + if (compression < 0 || compression > 9) + return BSP_ERROR_INVALID_INPUT; + + source_header = input_header(input); + context.data_types = + cJSON_GetObjectItemCaseSensitive(target_header, "data_types"); + format_json = cJSON_GetObjectItemCaseSensitive(source_header, "format"); + source_format = cJSON_GetStringValue(format_json); + target_format = cJSON_GetStringValue( + cJSON_GetObjectItemCaseSensitive(target_header, "format")); + if (source_format == NULL || target_format == NULL) + die("header has no format string"); + if (header_has_fill(source_header)) { + if (read_fill_value(input, &fill_value) != BSP_SUCCESS) + die("cannot read input fill_value"); + owns_fill = true; + } + source_is_tensor = strcmp(source_format, "custom") == 0; + + if (source_is_tensor) { + tensor = bsp_read_tensor(input, NULL); + if (tensor.level == NULL) + die("failed to parse input tensor"); + tensor.is_iso = header_values_are_iso(source_header); + rank = tensor.rank; + source_values = bsp_get_tensor_values(tensor); + iso = tensor.is_iso; + entries = tensor_to_coo(&tensor); + } else { + if (bsp_read_matrix(&matrix, input, NULL) != BSP_SUCCESS) + die("failed to parse input matrix"); + rank = bsp_matrix_format_is_vector(matrix.format) ? 1 : 2; + source_values = matrix.values; + iso = matrix.is_iso; + entries = matrix_to_coo(&matrix); + } + + size_t allocated_rank = rank ? (size_t) rank : 1; + source_transpose = malloc(allocated_rank * sizeof(*source_transpose)); + target_transpose = malloc(allocated_rank * sizeof(*target_transpose)); + stored_dims = malloc(allocated_rank * sizeof(*stored_dims)); + if (!source_transpose || !target_transpose || !stored_dims) + die("out of memory"); + for (int d = 0; d < rank; ++d) { + source_transpose[d] = + source_is_tensor && tensor.transpose ? tensor.transpose[d] : (size_t) d; + target_transpose[d] = (size_t) d; + } + if (!source_is_tensor && rank == 2 && + (matrix.format == BSP_DMATC || matrix.format == BSP_CSC || + matrix.format == BSP_DCSC || matrix.format == BSP_COOC)) { + source_transpose[0] = 1; + source_transpose[1] = 0; + } + + if (!strcmp(target_format, "custom")) { + custom = cJSON_GetObjectItemCaseSensitive(target_header, "custom"); + description = cJSON_GetObjectItemCaseSensitive(custom, "level"); + cJSON* transpose = cJSON_GetObjectItemCaseSensitive(custom, "transpose"); + if (!cJSON_IsObject(description)) + die("custom target requires custom.level"); + if (transpose != NULL) { + if (cJSON_GetArraySize(transpose) != rank) + die("transpose rank mismatch"); + for (int d = 0; d < rank; ++d) + target_transpose[d] = + (size_t) cJSON_GetNumberValue(cJSON_GetArrayItem(transpose, d)); + } + description = cJSON_Duplicate(description, 1); + } else { + description = predefined_level(target_format, target_transpose, rank); + if (description == NULL) + die("unknown or rank-incompatible target format"); + } + + apply_transpose(&entries, source_transpose, target_transpose); + for (int d = 0; d < rank; ++d) { + size_t logical = target_transpose[d]; + stored_dims[d] = source_is_tensor + ? tensor.dims[logical] + : (logical == 0 ? matrix.nrows : matrix.ncols); + } + values = values_in_entry_order(source_values, &entries, iso); + bsp_tensor_t result = bsp_construct_default_tensor_t(); + result.rank = rank; + result.nnz = entries.size; + result.is_iso = iso; + result.structure = source_is_tensor ? tensor.structure : matrix.structure; + result.dims = malloc(allocated_rank * sizeof(*result.dims)); + result.transpose = malloc(allocated_rank * sizeof(*result.transpose)); + if (!result.dims || !result.transpose) + die("out of memory"); + for (int d = 0; d < rank; ++d) { + result.dims[d] = source_is_tensor ? tensor.dims[d] + : (d == 0 ? matrix.nrows : matrix.ncols); + result.transpose[d] = target_transpose[d]; + } + root_ptr[0] = 0; + root_ptr[1] = entries.size; + result.level = build_level(&context, description, &entries, stored_dims, 0, + root_ptr, 1, true, values); + if (!strcmp(target_format, "custom")) { + cJSON* user = cJSON_CreateObject(); + if (bsp_write_tensor_cjson(output, result, NULL, user, compression) != + BSP_SUCCESS) + die("failed to write output tensor"); + cJSON_Delete(user); + } else if (write_predefined(output, result, target_format, compression) != + BSP_SUCCESS) { + die("failed to write predefined output"); + } + complete_output(output, target_header, owns_fill ? &fill_value : NULL); + + bsp_destroy_tensor_t(result); + free_entries(&entries); + if (source_is_tensor) + bsp_destroy_tensor_t(tensor); + else + bsp_destroy_matrix_t(&matrix); + if (owns_fill) + bsp_destroy_array_t(&fill_value); + free(source_transpose); + free(target_transpose); + free(stored_dims); + cJSON_Delete(description); + cJSON_Delete(source_header); + return BSP_SUCCESS; +} diff --git a/test/compliance/reformat.h b/test/compliance/reformat.h new file mode 100644 index 0000000..88cad4e --- /dev/null +++ b/test/compliance/reformat.h @@ -0,0 +1,63 @@ +/* + * SPDX-FileCopyrightText: 2026 Binsparse Developers + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct { + size_t* coord; + size_t value; +} entry_t; + +typedef struct { + entry_t* entry; + size_t size; + size_t capacity; + int rank; +} entries_t; + +typedef struct { + cJSON* data_types; +} reformat_context_t; + +void die(const char* message); +cJSON* read_json(const char* path); +cJSON* input_header(const char* path); +bool header_values_are_iso(cJSON* header); +bool header_has_fill(cJSON* header); +void canonicalize_predefined_alias(cJSON* header); +void complete_output(const char* path, cJSON* requested, + const bsp_array_t* fill_value); + +bsp_type_t type_from_name(const char* name); +void push_entry(entries_t* entries, const size_t* coord, size_t value); +void free_entries(entries_t* entries); +entries_t tensor_to_coo(const bsp_tensor_t* tensor); +entries_t matrix_to_coo(bsp_matrix_t* matrix); +void apply_transpose(entries_t* entries, const size_t* source, + const size_t* target); +bsp_array_t values_in_entry_order(const bsp_array_t source, + const entries_t* entries, bool iso); + +cJSON* predefined_level(const char* format, size_t* transpose, int rank); +bsp_level_t* build_level(const reformat_context_t* context, cJSON* description, + const entries_t* entries, const size_t* dims, + size_t depth, const size_t* parent_ptr, size_t parents, + bool root, bsp_array_t values); +bsp_error_t write_predefined(const char* path, bsp_tensor_t tensor, + const char* format, int compression); + +bsp_error_t bsp_reformat_file(const char* input, const char* output, + cJSON* target_header, int compression); diff --git a/test/compliance/run-binsparse-tests.sh b/test/compliance/run-binsparse-tests.sh new file mode 100755 index 0000000..23fe6dd --- /dev/null +++ b/test/compliance/run-binsparse-tests.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: 2026 Binsparse Developers +# SPDX-License-Identifier: BSD-3-Clause + +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd -- "${script_dir}/../.." && pwd)" +build_dir="${BINSPARSE_BUILD_DIR:-${repo_root}/build-compliance}" +tests_dir="${BINSPARSE_TESTS_DIR:-${build_dir}/binsparse-tests}" +tests_ref="${BINSPARSE_TESTS_REF:-9df16f6e7fe6d73c147ea5a3c23cf203876386b6}" +build_config="${BINSPARSE_BUILD_CONFIG:-Release}" + +for command in git pixi; do + if ! command -v "${command}" >/dev/null 2>&1; then + echo "error: required command '${command}' was not found" >&2 + exit 1 + fi +done + +if [[ -d "${tests_dir}/.git" ]]; then + git -C "${tests_dir}" fetch --depth 1 origin "${tests_ref}" + git -C "${tests_dir}" checkout --detach FETCH_HEAD +elif [[ -e "${tests_dir}" ]]; then + echo "error: ${tests_dir} exists but is not a Git checkout" >&2 + exit 1 +else + git clone --depth 1 https://github.com/Binsparse/binsparse-tests.git \ + "${tests_dir}" + git -C "${tests_dir}" fetch --depth 1 origin "${tests_ref}" + git -C "${tests_dir}" checkout --detach FETCH_HEAD +fi + +( + cd "${tests_dir}" + pixi install -e test +) + +executable_dir="${build_dir}/test/compliance" +if [[ -d "${executable_dir}/${build_config}" ]]; then + executable_dir="${executable_dir}/${build_config}" +fi + +for executable in npy_to_binsparse binsparse_to_npy binsparse_to_binsparse; do + if [[ ! -x "${executable_dir}/${executable}" ]]; then + echo "error: missing executable ${executable_dir}/${executable}" >&2 + exit 1 + fi +done + +export BINSPARSE_BIN="${executable_dir}" +( + cd "${tests_dir}" + pixi run -e test pytest -m hdf5 binsparse_tests/ "$@" +)